Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
Merge pull request #691 from keemsisi/fix/serve-state-size-gate
Nandakishor5a4990774baf
diff --cc tests/test_serve.pyindex e4b38187,bbdc6ea6..35f89395--- a/tests/test_serve.py+++ b/tests/test_serve.py@@@ -1037,240 -943,265 +1039,502 @@@ def test_resolve_max_token_budget_fallbassert "invalid LAYA_MAX_TOKEN_BUDGET" in caplog.textassert "LAYA_MAX_TOKEN_BUDGET must be positive" in caplog.text+BATCH_REQ = {+ "states": ["first state", "second state"],+ "questions": REQ["questions"],+}+++def test_batch_happy_path(monkeypatch):+ client, fake = _client(monkeypatch)+ r = client.post("/v1/systemone/batch", json=BATCH_REQ)+ assert r.status_code == 200, r.text+ data = r.json()+ assert "results" in data+ assert len(data["results"]) == 2+ assert "total_usage" in data+ assert data["total_usage"]["input_tokens"] == sum(+ res["usage"]["input_tokens"] for res in data["results"]+ )+ assert len(fake.calls) == 2+ assert "Server-Timing" in r.headers+ assert "X-Inference-Time-Ms" in r.headers+++def test_batch_missing_state_in_list_returns_400(monkeypatch):+ """A None state inside states list must be rejected with 400 'state' is required."""+ client, _ = _client(monkeypatch)+ bad_req = {"states": [None, "hello"], "questions": REQ["questions"]}+ r = client.post("/v1/systemone/batch", json=bad_req)+ assert r.status_code == 400+ assert "'state' is required" in r.json()["detail"]+++def test_batch_too_many_options_returns_413(monkeypatch):+ """Questions with more than MAX_CHOICE_OPTIONS must be rejected with 413."""+ client, _ = _client(monkeypatch)+ bad_questions = {+ "dept": {+ "type": "choice",+ "instructions": "which?",+ "criteria": {f"opt_{i}": f"desc_{i}" for i in range(101)},+ }+ }+ r = client.post("/v1/systemone/batch", json={"states": ["state"], "questions": bad_questions})+ assert r.status_code == 413+ assert "too many choice options" in r.json()["detail"]+++def test_batch_empty_states_returns_400(monkeypatch):+ client, _ = _client(monkeypatch)+ r = client.post("/v1/systemone/batch", json={"states": [], "questions": REQ["questions"]})+ assert r.status_code == 400+ assert "states" in r.json()["detail"]+++def test_batch_missing_states_returns_400(monkeypatch):+ client, _ = _client(monkeypatch)+ r = client.post("/v1/systemone/batch", json={"questions": REQ["questions"]})+ assert r.status_code == 400+++def test_batch_missing_questions_returns_400(monkeypatch):+ client, _ = _client(monkeypatch)+ r = client.post("/v1/systemone/batch", json={"states": ["state 1"]})+ assert r.status_code == 400+++def test_batch_too_many_states_returns_413(monkeypatch):+ client, _ = _client(monkeypatch)+ from laya.serve import MAX_BATCH_STATES+ oversized = {"states": ["state"] * (MAX_BATCH_STATES + 1), "questions": REQ["questions"]}+ r = client.post("/v1/systemone/batch", json=oversized)+ assert r.status_code == 413+ assert "too many states" in r.json()["detail"]+++def test_batch_individual_oversized_state_returns_413(monkeypatch):+ client, _ = _client(monkeypatch)+ from laya.serve import MAX_STATE_CHARS+ bad_req = {+ "states": ["ok", "X" * (MAX_STATE_CHARS + 10)],+ "questions": REQ["questions"],+ }+ r = client.post("/v1/systemone/batch", json=bad_req)+ assert r.status_code == 413+ assert "state too large" in r.json()["detail"]+++def test_batch_auth_required_when_key_set(monkeypatch):+ client, _ = _client(monkeypatch, api_key="secret123")+ r = client.post("/v1/systemone/batch", json=BATCH_REQ)+ assert r.status_code == 401+ ok = client.post(+ "/v1/systemone/batch",+ json=BATCH_REQ,+ headers={"Authorization": "Bearer secret123"},+ )+ assert ok.status_code == 200+++def test_batch_admission_bound_refuses_with_503(monkeypatch):+ """Batch endpoint must respect the admission bound and return 503 when busy."""+ import asyncio+ import httpx++ monkeypatch.delenv("LAYA_API_KEY", raising=False)+ monkeypatch.setenv("LAYA_MAX_CONCURRENT", "1")+ fake = GatedRouter()+ app = create_app(router=fake)+ seen = {}++ async def drive():+ transport = httpx.ASGITransport(app=app)+ async with httpx.AsyncClient(transport=transport, base_url="http://t") as client:+ first = asyncio.ensure_future(client.post("/v1/systemone", json=REQ))+ for _ in range(200):+ if fake.entered.is_set():+ break+ await asyncio.sleep(0.05)+ assert fake.entered.is_set(), "first request never reached inference"+ await asyncio.sleep(0.2)+ seen["batch_second"] = (await client.post("/v1/systemone/batch", json=BATCH_REQ)).status_code+ fake.release.set()+ seen["first"] = (await first).status_code++ asyncio.run(drive())++ assert seen["batch_second"] == 503, seen+ assert seen["first"] == 200, seen+++class BatchCapableFakeRouter(FakeRouter):+ """FakeRouter that implements predict_batch."""++ def __init__(self):+ super().__init__()+ self.batch_calls = []++ def predict_batch(self, requests, batch_size=None):+ self.batch_calls.append(requests)+ return [+ {+ "model": "laya-rl-agent",+ "answers": {+ "dept": {+ "type": "choice",+ "choice": "billing",+ "probabilities": {"billing": 0.94, "tech": 0.06},+ "confidence": 0.94,+ }+ },+ "usage": {"input_tokens": 42, "output_tokens": 0},+ "routing": {"model": "english", "reason": "English Latin text"},+ }+ for _ in requests+ ]+++def test_batch_uses_predict_batch_when_available(monkeypatch):+ monkeypatch.delenv("LAYA_API_KEY", raising=False)+ fake = BatchCapableFakeRouter()+ client = TestClient(create_app(router=fake))+ r = client.post("/v1/systemone/batch", json=BATCH_REQ)+ assert r.status_code == 200, r.text+ assert len(fake.batch_calls) == 1+ assert len(fake.batch_calls[0]) == 2+ assert len(fake.calls) == 0 # Confirms no sequential predict() fallback calls were made+def test_an_unpaired_surrogate_is_a_caller_error_not_a_server_fault():+ r"""A `\udXXX` escape with no pair parses as JSON and then cannot be UTF-8 encoded.++ The tokenizer raised `TypeError` from inside `build_sequence`, and `serve` maps only+ `ValueError` to 422, so a malformed string in the client's own body came back as a 500+ "inference failed" -- a server fault plus an operator traceback for the caller's mistake.+ A *paired* surrogate is an ordinary astral character (an emoji) and must keep working.+ """+ seen = []++ class RecordingRouter:+ loaded = ["english"]++ def predict(self, state, questions, model=None):+ seen.append(state)+ return {"model": "stub", "answers": {}, "usage": {}}++ client = TestClient(create_app(router=RecordingRouter()), raise_server_exceptions=False)++ lone = [+ b'{"state":"\\ud800","questions":{"q":{"type":"noul","instructions":"x"}}}',+ b'{"state":"ok","questions":{"q":{"type":"noul","instructions":"\\udfff"}}}',+ b'{"state":"ok","questions":{"q":{"type":"choice","instructions":"x",'+ b'"criteria":["\\ud800","b"]}}}',+ ]+ for body in lone:+ res = client.post("/v1/systemone", content=body,+ headers={"content-type": "application/json"})+ assert res.status_code == 400, (body, res.status_code, res.text)+ assert "unpaired surrogate" in res.text, res.text++ # a paired surrogate is one astral character by the time json.loads is done: it must reach+ # the router rather than be rejected+ seen.clear()+ ok = json.dumps({"state": "hi \U0001f600",+ "questions": {"q": {"type": "noul", "instructions": "x"}}}).encode("utf-8")+ res = client.post("/v1/systemone", content=ok,+ headers={"content-type": "application/json"})+ assert res.status_code == 200, (res.status_code, res.text)+ assert seen and "\U0001f600" in str(seen[0]), seen+++def test_a_deeply_nested_state_is_not_a_recursion_error():+ """`json.loads` accepts nesting far deeper than Python's recursion limit.++ A recursive walk over the parsed body therefore turned a body the parser handles into a+ `RecursionError`, i.e. one 500 replaced by another. The walk uses an explicit stack, so the+ depth the JSON parser accepts is the depth this handles.+ """+ class NestedRouter:+ loaded = ["english"]++ def predict(self, state, questions, model=None):+ return {"model": "stub", "answers": {}, "usage": {}}++ client = TestClient(create_app(router=NestedRouter()), raise_server_exceptions=False)+ questions = {"q": {"type": "noul", "instructions": "x"}}++ for depth in (200, 600, 1200):+ body = json.dumps({"state": ["x" * 3] * depth, "questions": questions}).encode("utf-8")+ res = client.post("/v1/systemone", content=body,+ headers={"content-type": "application/json"})+ assert res.status_code == 200, (depth, res.status_code, res.text)+def test_inference_timing_headers_absent_on_error():+ """Inference timing headers must not be attached to error responses."""+ router = FakeRouter()+ client = TestClient(create_app(router=router))+ res = client.post("/v1/systemone", json={"state": "missing questions"})+ assert res.status_code == 400+ assert "Server-Timing" not in res.headers+ assert "X-Inference-Time-Ms" not in res.headers+++ # --------------------------------------------------------------------------- #+ # The state-size gate has to measure the text that reaches the tokenizer+ # --------------------------------------------------------------------------- #++ # Two dict states whose `str()` and whose `serialize_state()` are different lengths, which is+ # what the gate measured wrongly. The numbers are exact rather than approximate: if either one+ # moves, the divergence these checks pin has moved with it.+ _QUOTE_STATE = {"body": '"' * 49988} # len(str()) == 50000, serialized == 99988+ _ZWSP_STATE = {"body": "\u200b" * 8332} # len(str()) == 50004, serialized == 8344+++ def _serialized_len(state):+ """The length `laya.common.serialize_state` produces, without importing torch."""+ return len(json.dumps(state, ensure_ascii=False))+++ def test_the_gate_counts_the_json_the_tokenizer_will_see(monkeypatch):+ """`repr` leaves a `"` inside a value one character; JSON escapes it to two.++ So a state sitting exactly on the 50000-character gate was tokenized as 99988 characters --+ the documented limit admitting very nearly twice what it says.+ """+ assert len(str(_QUOTE_STATE)) == MAX_STATE_CHARS # exactly at a repr-measured gate+ assert _serialized_len(_QUOTE_STATE) == 99988 # what build_sequence would encode+ client, fake = _client(monkeypatch)+ r = client.post("/v1/systemone", json={**REQ, "state": _QUOTE_STATE})+ assert r.status_code == 413+ assert r.json()["detail"] == "state too large (99988 > %d chars)" % MAX_STATE_CHARS+ assert fake.calls == [] # refused before inference, as intended+++ def test_a_state_a_sixth_of_the_limit_is_not_refused_for_its_repr(monkeypatch):+ """The same mismatch the other way: `repr` escapes a zero-width space to six characters.++ `ensure_ascii=False` writes the one character it is, so this state serializes to 8344 --+ and was answered "state too large (50004 > 50000 chars)".+ """+ assert len(str(_ZWSP_STATE)) == 50004 # over a repr-measured gate+ assert _serialized_len(_ZWSP_STATE) == 8344 # a sixth of the limit, once serialized+ client, fake = _client(monkeypatch)+ r = client.post("/v1/systemone", json={**REQ, "state": _ZWSP_STATE})+ assert r.status_code == 200+ assert fake.calls[0]["state"] == _ZWSP_STATE # reached inference unchanged+++ # Deliberately states whose `repr` and whose JSON are the same length, so these four answer the+ # same before and after the fix. They are here to pin what must NOT move: the limit still admits a+ # state that sits exactly on it and still refuses one character more.+ # The explicit ids are required, not cosmetic. pytest builds each case's node id out of the+ # parameter value, and writes that node id into the PYTEST_CURRENT_TEST environment variable -- a+ # 50 000-character state produces a 50 084-character id, and a Windows environment variable cannot+ # exceed 32 767 characters, so every case errors at teardown with+ # `ValueError: the environment variable is longer than 32767 characters`. It passes on Linux, which+ # has no comparable per-variable limit. `b"[" * 100000` in this file carries an id for the same+ # reason.+ @pytest.mark.parametrize("state, expected", [+ # str() and JSON are both exactly 50000+ pytest.param({"body": "a" * 49988}, 200, id="dict-exactly-at-the-limit"),+ # both exactly one character over+ pytest.param({"body": "a" * 49989}, 413, id="dict-one-character-over"),+ # a string state is its own serialization+ pytest.param("A" * MAX_STATE_CHARS, 200, id="str-exactly-at-the-limit"),+ pytest.param("A" * (MAX_STATE_CHARS + 1), 413, id="str-one-character-over"),+ ])+ def test_the_declared_limit_is_the_limit_for_ordinary_states(monkeypatch, request, state, expected):+ # Asserted here rather than left to the Windows job: pytest writes this node id into+ # PYTEST_CURRENT_TEST, where Windows caps a variable at 32 767 characters, so dropping the+ # explicit ids above turns every case in this test into a teardown error -- on Windows only.+ # This assertion fails on any platform, so the ids cannot be lost without CI saying so.+ assert len(request.node.nodeid) < 32767, \+ "node id is %d characters; give this parameter an explicit short id" % len(request.node.nodeid)+ client, _ = _client(monkeypatch)+ r = client.post("/v1/systemone", json={**REQ, "state": state})+ assert r.status_code == expected+++ def test_the_gate_agrees_with_the_function_that_feeds_the_tokenizer():+ """Pin serve.py's inline `json.dumps` to `serialize_state` itself.++ serve.py cannot import it -- `laya.common` imports torch at module level and+ `import laya.serve` must not (tests/test_lazy_import.py) -- so the mirror is held here+ instead. Looked up rather than imported at module scope so that a serve.py without the+ helper fails this one check instead of aborting the suite's import.+ """+ import laya.serve as serve_mod++ state_length = getattr(serve_mod, "_state_length", None)+ assert state_length is not None, "laya.serve no longer measures the state through _state_length"+ serialize_state = pytest.importorskip("laya.common").serialize_state+ for state in (_QUOTE_STATE, _ZWSP_STATE, "plain text", {}, [],+ {"a": {"b": ["c", 1, 1.5, None, True]}},+ ["\u4e2d\u6587", '"', "'", "\n", "\x00", "\U0001f600"],+ # A non-BMP format character is the widest divergence a client can send: `repr`+ # writes it as the ten characters `\U000e0001`, `ensure_ascii=False` as one.+ {"tag": "\U000e0001" * 10},+ {"d": "it's", "q": 'say "hi"'}):+ assert state_length(state) == len(serialize_state(state)), state+++ @pytest.fixture+ def default_int_digits():+ """Pin CPython's int-to-str digit guard for the tests that rely on it.++ The guard defaults to 4300 but is operator-configurable (`PYTHONINTMAXSTRDIGITS`,+ `sys.set_int_max_str_digits`). With it switched off, `json.dumps(10 ** 4300)` succeeds and+ `json.loads` accepts a 4301-digit body, so these two tests fail while the code under test stays+ correct -- verified: `PYTHONINTMAXSTRDIGITS=0` turns both of them red on an unmodified tree. A+ test that depends on an operator's setting is measuring the environment, not the change.+ """+ previous = sys.get_int_max_str_digits()+ sys.set_int_max_str_digits(4300)+ try:+ yield+ finally:+ sys.set_int_max_str_digits(previous)+++ def test_an_unserializable_state_is_a_400_not_a_count_nothing_measured(default_int_digits):+ """The old `except Exception` answered 413 "state too large (50001 > 50000 chars)".++ That reported a length nothing had measured, for a state that may be a few bytes. These+ states are unreachable over HTTP -- `json.loads` builds only JSON types -- but they are+ what an in-process caller of this helper can pass, and on the old path a state+ `serialize_state` cannot render passed the gate and failed later as a 500.+ """+ from fastapi import HTTPException++ # 10 ** 4300 has 4301 digits, one past CPython's int-to-str limit, and is built by+ # arithmetic because `int("1" * 4301)` would hit that same limit while parsing.+ for state in ({1, 2}, 10 ** 4300, {"n": 10 ** 4300}, [10 ** 4300]):+ with pytest.raises(HTTPException) as exc:+ _check_request_limits(state, REQ["questions"])+ assert exc.value.status_code == 400+ # The whole detail, so a count nothing measured cannot creep back into it.+ assert exc.value.detail == "'state' must be JSON-serializable"+++ def test_an_oversized_integer_state_never_reaches_the_gate(monkeypatch, default_int_digits):+ """Why the 400 above is not an HTTP-reachable case: the parser refuses the body first."""+ client, fake = _client(monkeypatch)+ r = client.post("/v1/systemone",+ content=b'{"state": ' + b"1" * 4301 + b', "questions": {}}',+ headers={"content-type": "application/json"})+ assert r.status_code == 400+ assert r.json()["detail"] == "request body must be valid JSON"+ assert fake.calls == []+++ # --- the lower-bound probe in front of the exact measurement ---------------------------------++ def test_the_probe_refuses_an_oversized_value_without_serializing_the_state(monkeypatch):+ """An oversized state is refused without the 2 MiB dump whose result cannot change the verdict.++ This is the check that goes red if `_state_length_lower_bound_over` is removed: without it the gate+ still answers 413, so the *absence of the dump* is the only observable difference.+ """+ import laya.serve as serve_mod++ calls = []+ real_dumps = json.dumps++ def counting_dumps(obj, **kw):+ calls.append(kw)+ return real_dumps(obj, **kw)++ # Scoped to the module's own binding: `serve_mod.json` IS the stdlib module, so patching an+ # attribute on it would replace `json.dumps` for the whole process for the test's duration.+ monkeypatch.setattr(serve_mod, "json", SimpleNamespace(dumps=counting_dumps))+ # F2: a conversation LIST is this library's dominant state shape, and the probe's list branch+ # can be deleted with every other check still green unless one of these is list-shaped.+ for state in ({"body": "a" * (2 * 1024 * 1024)},+ ["z" * 60000],+ [{"role": "user", "content": "x" * 70000}],+ ({"role": "user", "content": "x" * 70000},)):+ calls.clear()+ assert serve_mod._state_length(state) > serve_mod.MAX_STATE_CHARS, state.__class__.__name__+ assert calls == [], "a %s state was serialized although one value already exceeds the cap" \+ % type(state).__name__+++ def test_the_probe_verdict_always_matches_the_exact_measurement():+ """The probe may only report "provably over"; its verdict must never differ from the encoder's.++ An unsound probe would be a security bug in the same family as the one this PR fixes, so the two+ are compared across shapes placed deliberately on both sides of the limit -- including the+ escape-heavy values where `repr` and JSON diverge, which is what the gate exists for.+ """+ import laya.serve as serve_mod+ cap = serve_mod.MAX_STATE_CHARS+ shapes = []+ for ch in ("a", '"', "\u200b", "\U000e0001", "\n", "\\", "\u4e2d"):+ for n in (1, 100, cap // 2, cap - 1, cap, cap + 1, cap * 2):+ shapes.append({"body": ch * n})+ shapes.append([ch * n])+ shapes.append({"a": {"b": [ch * n]}})+ shapes += [+ {("k%05d" % i): "v" for i in range(3000)},+ {("k%05d" % i): "v" * 40 for i in range(3000)},+ [{"role": "user", "content": "hi"} for _ in range(4000)],+ {"n": 1, "f": 1.5, "t": True, "z": None},+ {},+ [],+ ]+ assert len(shapes) == 7 * 7 * 3 + 6, len(shapes)+ for state in shapes:+ exact = len(json.dumps(state, ensure_ascii=False))+ probed = serve_mod._state_length(state)+ assert (probed > cap) == (exact > cap), \+ "probe and encoder disagree for a %d-character state" % exact+ if exact <= cap:+ # Stronger than agreeing on the verdict: under the cap the *exact* length is returned,+ # so the 413 message keeps quoting a number something actually measured.+ assert probed == exact+++ def test_the_early_refusal_reports_a_number_it_measured():+ """The 413 must not invent a count. Reporting `cap + 1` would answer "50001 > 50000" for a+ 60 012-character state -- the same fabricated count this gate was written to remove.+ """+ import laya.serve as serve_mod+ cap = serve_mod.MAX_STATE_CHARS+ # `cap + 1` is NOT the tell: {"body": "a" * 50001} legitimately sums to exactly that. The+ # property that distinguishes a measured bound from a fabricated one is `cap < reported <= exact`.+ for state in ({"body": "x" * 60000}, {"body": "y" * (2 * 1024 * 1024)}, {"a": ["z" * 80000]},+ {"body": "a" * (cap + 1)}, ["w" * 60000]):+ exact = len(json.dumps(state, ensure_ascii=False))+ reported = serve_mod._state_length(state)+ assert reported > cap, reported+ assert reported <= exact, \+ "reported %d for a state of %d characters -- a 413 must never overstate" % (reported, exact)+++ def test_the_probe_is_bounded_and_falls_through_rather_than_guessing():+ """The probe examines a constant number of values, so it can never become the expensive step.++ A state of 200 000 small values is not provably over within that budget, and must fall through+ to the encoder rather than be refused on a partial sum.+ """+ import laya.serve as serve_mod+ assert serve_mod._STATE_PROBE_VALUES <= 256+ wide = {("k%06d" % i): "x" for i in range(200000)}+ assert serve_mod._state_length_lower_bound_over(wide, serve_mod.MAX_STATE_CHARS) == 0+ assert serve_mod._state_length(wide) == len(json.dumps(wide, ensure_ascii=False))+++ def test_the_probe_walks_exact_container_types_only():+ """A subclass may override `values()` while `json.dumps` reads the real items, so walking one+ would let the "lower bound" exceed the true length. Measured before this was narrowed: a+ 13-character state refused as `60000 > 50000`. No HTTP request can reach it -- `json.loads`+ builds exact types -- but the 400 branch exists for in-process callers.+ """+ import laya.serve as serve_mod++ class Lying(dict):+ def values(self):+ return ["x" * 60000]++ state = Lying(x="tiny")+ exact = len(json.dumps(state, ensure_ascii=False))+ assert serve_mod._state_length_lower_bound_over(state, serve_mod.MAX_STATE_CHARS) == 0+ assert serve_mod._state_length(state) == exact == 13