Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(serve): measure the state limit on the text the tokenizer receives
Adeshina Lasisid8038b8e2208
diff --git a/docs/http-api.md b/docs/http-api.mdindex a1bb894a..5229fc75 100644--- a/docs/http-api.md+++ b/docs/http-api.md@@ -132,7 +132,7 @@ nothing but the bytes it read. Every one of them is a `413`; the `detail` says w| limit | value ||---|---|| request body | 2 MiB, enforced while streaming -- a chunked or understated `Content-Length` cannot bypass it |-| `state` | 50,000 characters |+| `state` | 50,000 characters of the text the model is given -- the string itself for a string state, `json.dumps(state, ensure_ascii=False)` for an object or array || questions per request | 64 || options per `choice` question | 100 || levels per `score` question | 32 |diff --git a/examples/server.py b/examples/server.pyindex aab5d01a..23bc3ec4 100644--- a/examples/server.py+++ b/examples/server.py@@ -67,6 +67,30 @@ MAX_CHOICE_OPTIONS = getattr(_laya_serve, "MAX_CHOICE_OPTIONS", 100)MAX_SCORE_LEVELS = getattr(_laya_serve, "MAX_SCORE_LEVELS", 32)MAX_TOTAL_OPTIONS = getattr(_laya_serve, "MAX_TOTAL_OPTIONS", 512)++def _fallback_state_length(state: Any) -> int:+ """`laya.serve._state_length` for a laya that predates it, 400 included.++ The 400 is part of what it does: without it an unserializable state raises a bare `TypeError`+ out of `_check_request_limits`, which `/predict` calls outside its own `try`, so a caller error+ would arrive as a 500 where it used to be answered.+ """+ if isinstance(state, str):+ return len(state)+ try:+ return len(json.dumps(state, ensure_ascii=False))+ except (TypeError, ValueError, RecursionError):+ raise HTTPException(status_code=400, detail="'state' must be JSON-serializable")+++# Borrowed, not restated: the state length has to be measured on the text the tokenizer sees,+# which is `serialize_state(state)` and not `str(state)`. On states a client can send the two+# differ by up to 2x upward (a `"` costs one character in `repr` and two in JSON) and 10x downward+# (`repr` escapes a zero-width space to six characters and a non-BMP format character to ten, where+# `ensure_ascii=False` writes the one character it is). See `laya.serve._state_length`. Same+# `getattr` shape as the bounds above.+_state_length = getattr(_laya_serve, "_state_length", _fallback_state_length)+# The demo answers failures the way laya.serve does: a fixed message to the caller, the# traceback to this logger. Without it a 500 arrived as a bare status line in the server# output and the cause had to be reproduced in-process to be found at all.@@ -268,9 +292,9 @@ def _check_request_limits(state: Any, questions: Dict[str, Any]) -> None:Laya encodes the state once per question, so cost is questions x state size,collated into one tensor, and a choice or score question adds one sequence per- option against a shared head budget. The state length is measured exactly as- laya.serve measures it -- `len(v)` for a string, `len(str(v))` for a dict or list- -- and the option counts exactly as it counts them, over `choice` and `score`+ option against a shared head budget. The state length comes from laya.serve's own+ `_state_length`, so the two surfaces cannot disagree about what a state measures,+ and the option counts exactly as it counts them, over `choice` and `score`criteria only.Checked here rather than declared as pydantic constraints on the request models,@@ -284,7 +308,7 @@ def _check_request_limits(state: Any, questions: Dict[str, Any]) -> None:status_code=413,detail="too many questions (%d > %d)" % (len(questions), MAX_QUESTIONS),)- size = len(state) if isinstance(state, str) else len(str(state))+ size = _state_length(state)if size > MAX_STATE_CHARS:raise HTTPException(status_code=413,@@ -3438,9 +3462,13 @@ async def gui_predict(request: Request) -> HTMLResponse:try:_check_request_limits(req.state, req.questions)except HTTPException as exc:- return _gui_error("Request too large", [str(exc.detail)],- f"The server takes up to {MAX_QUESTIONS} questions and a state of up to "- f"{MAX_STATE_CHARS:,} characters. Split the request or shorten the state.")+ # Branch on the status: this check answers 400 as well as 413 now, and a size hint under+ # the heading "Request too large" would describe the wrong problem.+ if exc.status_code == 413:+ return _gui_error("Request too large", [str(exc.detail)],+ f"The server takes up to {MAX_QUESTIONS} questions and a state of up to "+ f"{MAX_STATE_CHARS:,} characters. Split the request or shorten the state.")+ return _gui_error("Bad request", [str(exc.detail)], "Fix the request and send it again.")questions = _questions(req.questions)try:started = time.perf_counter()diff --git a/laya/serve.py b/laya/serve.pyindex ed08d39a..588ccd21 100644--- a/laya/serve.py+++ b/laya/serve.py@@ -199,6 +199,52 @@ def _resolve_port() -> int:return port+def _state_length(state: Any) -> int:+ r"""Length of the state text that will be tokenized; 400 if it has none.++ `laya.common.serialize_state(state)` is what gets tokenized -- the state itself for a string,+ `json.dumps(state, ensure_ascii=False)` for a dict or list, which the agents then strip mask+ tokens out of, and that can only shorten it. `MAX_STATE_CHARS` has to be measured on that+ text, not on `str()`, which is a different length in both directions:++ * `repr` quotes a value with `'` and leaves a `"` inside it one character, where JSON escapes+ it to the two characters `\"`. A `{"body": '"' * 49988}` state has a `len(str())` of exactly+ 50000, so it passed the gate, and then serialized to 99988 -- the documented 50000-character+ limit admitting very nearly twice what it says.+ * `repr` renders a zero-width space as the six characters `\u200b` where `ensure_ascii=False`+ writes the one character it is, so a state that serializes to 8344 characters -- a sixth of+ the limit -- was refused with "state too large (50004 > 50000 chars)".++ `json.dumps` inline rather than importing `serialize_state`: `laya.common` imports torch at+ module level and `import laya.serve` must not (tests/test_lazy_import.py). The two are pinned+ to each other in tests/test_serve.py.++ The dump runs before the gate decides, so its bound is MAX_BODY_BYTES, not MAX_STATE_CHARS. At+ the gate's own ceiling it is 0.133 ms against 0.080 ms for the `str()` it replaces, three+ orders of magnitude under the forward pass that state then gets. On a 2 MiB body -- the largest+ the streaming cap allows, and one this refuses -- it is 5.68 ms against 3.46 ms, on a request+ that is rejected either way and whose `str()` upstream already walked in full. (Best of 200+ after 20 warm-up iterations, CPython 3.12 on a 10-core arm64 laptop.) So the serialized text is+ not worth threading through `predict()` to be encoded only once.+ """+ from fastapi import HTTPException++ if isinstance(state, str):+ return len(state) # `serialize_state` returns a string state unchanged+ try:+ return len(json.dumps(state, ensure_ascii=False))+ except (TypeError, ValueError, RecursionError):+ # A state `serialize_state` cannot render is not a size problem, and the 413 this replaces+ # reported a count nothing had measured: `str()` raises on an integer of over 4300 digits+ # (CPython's own int-to-str guard), and the old `except` answered "state too large (50001+ # > 50000 chars)" for a state of a few kilobytes. No HTTP request reaches that case --+ # `json.loads` builds only JSON types and refuses that integer itself, which+ # `_systemone_inner` already answers 400 -- but in-process callers get here, and a state+ # `json.dumps` refuses (a set, a datetime, a circular reference) used to pass the gate on+ # its `str()` and then fail inside `serialize_state`, reported as a 500 "inference failed".+ raise HTTPException(status_code=400, detail="'state' must be JSON-serializable")++def _check_request_limits(state: Any, questions: Any) -> None:"""Reject absent or oversized inference requests before tokenization (400/413)."""from fastapi import HTTPException@@ -245,10 +291,7 @@ def _check_request_limits(state: Any, questions: Any) -> None:detail="too many answer options across questions (%d > %d)" % (total_options, MAX_TOTAL_OPTIONS),)- try:- state_len = len(state) if isinstance(state, str) else len(str(state))- except Exception:- state_len = MAX_STATE_CHARS + 1+ state_len = _state_length(state)if state_len > MAX_STATE_CHARS:raise HTTPException(status_code=413,detail="state too large (%d > %d chars)" % (state_len, MAX_STATE_CHARS))diff --git a/tests/test_example_server_limits.py b/tests/test_example_server_limits.pyindex 59781e6c..a1cc7845 100644--- a/tests/test_example_server_limits.py+++ b/tests/test_example_server_limits.py@@ -261,11 +261,30 @@ def main():+ [("n%d" % i, noul_crit) for i in range(30)])),("noul only", "hi", {"n%d" % i: noul_crit for i in range(10)}),("an ordinary request", {"body": "billed twice"}, one),+ # A state whose `str()` and whose JSON serialization are different lengths (50000 vs 99988).+ # This case catches the two surfaces DIVERGING -- one measuring the serialized text and the+ # other `str(state)`. It cannot catch them being wrong together: `examples/server.py` binds+ # `_state_length` from `laya.serve` by `getattr`, so a regression inside that helper moves+ # both and parity stays green. Measured: regressing `_state_length` fails 0 of these parity+ # checks and 4 of the absolute ones below. Those are the real coverage -- do not prune them+ # as redundant.+ ("a state whose repr is half its JSON", {"body": '"' * 49988}, one),]for label, st, qs in parity_cases:s, d = serve_verdict(st, qs), demo_verdict(st, qs)ok("parity with laya.serve: %s" % label, s == d, "serve=%s demo=%s" % (s, d))+ # Parity alone cannot see a bug both surfaces share, and `getattr` guarantees they share one:+ # with `_state_length` measuring `str(state)` again, both agree on accepting a state that+ # serializes to 99 988 characters and every parity check above stays green (measured: 0 of them+ # fail, 4 of these do). So the verdict itself is asserted, not just the agreement.+ quote_heavy = {"body": '"' * 49988}+ assert len(str(quote_heavy)) <= MAX_STATE_CHARS, "the fixture must pass a str()-based gate"+ assert len(json.dumps(quote_heavy, ensure_ascii=False)) > MAX_STATE_CHARS, "and fail a real one"+ for who, verdict in (("laya.serve", serve_verdict(quote_heavy, one)),+ ("the demo server", demo_verdict(quote_heavy, one))):+ ok("%s refuses a state whose JSON is twice its repr" % who, verdict == "refused", verdict)+# --- the resident-checkpoint cap: derived, not copied -------------------# examples/server.py used to build its Router with `max_loaded=1`, a copy of a default# laya/router.py retired in #180. A cap of one cannot hold both english anddiff --git a/tests/test_serve.py b/tests/test_serve.pyindex dab066ab..1627fb1f 100644--- a/tests/test_serve.py+++ b/tests/test_serve.py@@ -20,7 +20,9 @@ ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))from laya.serve import ( # noqa: E402DEFAULT_MAX_TOKEN_BUDGET,MAX_BODY_BYTES,+ MAX_STATE_CHARS,_apply_thread_limit,+ _check_request_limits,_env_bool,_resolve_max_token_budget,_resolve_max_loaded,@@ -941,3 +943,134 @@ def test_resolve_max_token_budget_fallback(monkeypatch, caplog):assert "invalid LAYA_MAX_TOKEN_BUDGET" in caplog.textassert "LAYA_MAX_TOKEN_BUDGET must be positive" in caplog.text++# --------------------------------------------------------------------------- #+# The state-size gate has to measure the text that reaches the tokenizer+# --------------------------------------------------------------------------- #++# Two dict states whose `str()` and whose `serialize_state()` are different lengths, which is+# what the gate measured wrongly. The numbers are exact rather than approximate: if either one+# moves, the divergence these checks pin has moved with it.+_QUOTE_STATE = {"body": '"' * 49988} # len(str()) == 50000, serialized == 99988+_ZWSP_STATE = {"body": "\u200b" * 8332} # len(str()) == 50004, serialized == 8344+++def _serialized_len(state):+ """The length `laya.common.serialize_state` produces, without importing torch."""+ return len(json.dumps(state, ensure_ascii=False))+++def test_the_gate_counts_the_json_the_tokenizer_will_see(monkeypatch):+ """`repr` leaves a `"` inside a value one character; JSON escapes it to two.++ So a state sitting exactly on the 50000-character gate was tokenized as 99988 characters --+ the documented limit admitting very nearly twice what it says.+ """+ assert len(str(_QUOTE_STATE)) == MAX_STATE_CHARS # exactly at a repr-measured gate+ assert _serialized_len(_QUOTE_STATE) == 99988 # what build_sequence would encode+ client, fake = _client(monkeypatch)+ r = client.post("/v1/systemone", json={**REQ, "state": _QUOTE_STATE})+ assert r.status_code == 413+ assert r.json()["detail"] == "state too large (99988 > %d chars)" % MAX_STATE_CHARS+ assert fake.calls == [] # refused before inference, as intended+++def test_a_state_a_sixth_of_the_limit_is_not_refused_for_its_repr(monkeypatch):+ """The same mismatch the other way: `repr` escapes a zero-width space to six characters.++ `ensure_ascii=False` writes the one character it is, so this state serializes to 8344 --+ and was answered "state too large (50004 > 50000 chars)".+ """+ assert len(str(_ZWSP_STATE)) == 50004 # over a repr-measured gate+ assert _serialized_len(_ZWSP_STATE) == 8344 # a sixth of the limit, once serialized+ client, fake = _client(monkeypatch)+ r = client.post("/v1/systemone", json={**REQ, "state": _ZWSP_STATE})+ assert r.status_code == 200+ assert fake.calls[0]["state"] == _ZWSP_STATE # reached inference unchanged+++# Deliberately states whose `repr` and whose JSON are the same length, so these four answer the+# same before and after the fix. They are here to pin what must NOT move: the limit still admits a+# state that sits exactly on it and still refuses one character more.[email protected]("state, expected", [+ ({"body": "a" * 49988}, 200), # str() and JSON are both exactly 50000+ ({"body": "a" * 49989}, 413), # both exactly one character over+ ("A" * MAX_STATE_CHARS, 200), # a string state is its own serialization+ ("A" * (MAX_STATE_CHARS + 1), 413),+])+def test_the_declared_limit_is_the_limit_for_ordinary_states(monkeypatch, state, expected):+ client, _ = _client(monkeypatch)+ r = client.post("/v1/systemone", json={**REQ, "state": state})+ assert r.status_code == expected+++def test_the_gate_agrees_with_the_function_that_feeds_the_tokenizer():+ """Pin serve.py's inline `json.dumps` to `serialize_state` itself.++ serve.py cannot import it -- `laya.common` imports torch at module level and+ `import laya.serve` must not (tests/test_lazy_import.py) -- so the mirror is held here+ instead. Looked up rather than imported at module scope so that a serve.py without the+ helper fails this one check instead of aborting the suite's import.+ """+ import laya.serve as serve_mod++ state_length = getattr(serve_mod, "_state_length", None)+ assert state_length is not None, "laya.serve no longer measures the state through _state_length"+ serialize_state = pytest.importorskip("laya.common").serialize_state+ for state in (_QUOTE_STATE, _ZWSP_STATE, "plain text", {}, [],+ {"a": {"b": ["c", 1, 1.5, None, True]}},+ ["\u4e2d\u6587", '"', "'", "\n", "\x00", "\U0001f600"],+ # A non-BMP format character is the widest divergence a client can send: `repr`+ # writes it as the ten characters `\U000e0001`, `ensure_ascii=False` as one.+ {"tag": "\U000e0001" * 10},+ {"d": "it's", "q": 'say "hi"'}):+ assert state_length(state) == len(serialize_state(state)), state+++def default_int_digits():+ """Pin CPython's int-to-str digit guard for the tests that rely on it.++ The guard defaults to 4300 but is operator-configurable (`PYTHONINTMAXSTRDIGITS`,+ `sys.set_int_max_str_digits`). With it switched off, `json.dumps(10 ** 4300)` succeeds and+ `json.loads` accepts a 4301-digit body, so these two tests fail while the code under test stays+ correct -- verified: `PYTHONINTMAXSTRDIGITS=0` turns both of them red on an unmodified tree. A+ test that depends on an operator's setting is measuring the environment, not the change.+ """+ previous = sys.get_int_max_str_digits()+ sys.set_int_max_str_digits(4300)+ try:+ yield+ finally:+ sys.set_int_max_str_digits(previous)+++def test_an_unserializable_state_is_a_400_not_a_count_nothing_measured(default_int_digits):+ """The old `except Exception` answered 413 "state too large (50001 > 50000 chars)".++ That reported a length nothing had measured, for a state that may be a few bytes. These+ states are unreachable over HTTP -- `json.loads` builds only JSON types -- but they are+ what an in-process caller of this helper can pass, and on the old path a state+ `serialize_state` cannot render passed the gate and failed later as a 500.+ """+ from fastapi import HTTPException++ # 10 ** 4300 has 4301 digits, one past CPython's int-to-str limit, and is built by+ # arithmetic because `int("1" * 4301)` would hit that same limit while parsing.+ for state in ({1, 2}, 10 ** 4300, {"n": 10 ** 4300}, [10 ** 4300]):+ with pytest.raises(HTTPException) as exc:+ _check_request_limits(state, REQ["questions"])+ assert exc.value.status_code == 400+ # The whole detail, so a count nothing measured cannot creep back into it.+ assert exc.value.detail == "'state' must be JSON-serializable"+++def test_an_oversized_integer_state_never_reaches_the_gate(monkeypatch, default_int_digits):+ """Why the 400 above is not an HTTP-reachable case: the parser refuses the body first."""+ client, fake = _client(monkeypatch)+ r = client.post("/v1/systemone",+ content=b'{"state": ' + b"1" * 4301 + b', "questions": {}}',+ headers={"content-type": "application/json"})+ assert r.status_code == 400+ assert r.json()["detail"] == "request body must be valid JSON"+ assert fake.calls == []