Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(serve): measure the state limit on the text the tokenizer receives

Adeshina Lasisid8038b8e2208
diff --git a/docs/http-api.md b/docs/http-api.md
index a1bb894a..5229fc75 100644
--- a/docs/http-api.md
+++ b/docs/http-api.md
@@ -132,7 +132,7 @@ nothing but the bytes it read. Every one of them is a `413`; the `detail` says w
| limit | value |
|---|---|
| request body | 2 MiB, enforced while streaming -- a chunked or understated `Content-Length` cannot bypass it |
-| `state` | 50,000 characters |
+| `state` | 50,000 characters of the text the model is given -- the string itself for a string state, `json.dumps(state, ensure_ascii=False)` for an object or array |
| questions per request | 64 |
| options per `choice` question | 100 |
| levels per `score` question | 32 |
diff --git a/examples/server.py b/examples/server.py
index aab5d01a..23bc3ec4 100644
--- a/examples/server.py
+++ b/examples/server.py
@@ -67,6 +67,30 @@ MAX_CHOICE_OPTIONS = getattr(_laya_serve, "MAX_CHOICE_OPTIONS", 100)
MAX_SCORE_LEVELS = getattr(_laya_serve, "MAX_SCORE_LEVELS", 32)
MAX_TOTAL_OPTIONS = getattr(_laya_serve, "MAX_TOTAL_OPTIONS", 512)
+
+def _fallback_state_length(state: Any) -> int:
+ """`laya.serve._state_length` for a laya that predates it, 400 included.
+
+ The 400 is part of what it does: without it an unserializable state raises a bare `TypeError`
+ out of `_check_request_limits`, which `/predict` calls outside its own `try`, so a caller error
+ would arrive as a 500 where it used to be answered.
+ """
+ if isinstance(state, str):
+ return len(state)
+ try:
+ return len(json.dumps(state, ensure_ascii=False))
+ except (TypeError, ValueError, RecursionError):
+ raise HTTPException(status_code=400, detail="'state' must be JSON-serializable")
+
+
+# Borrowed, not restated: the state length has to be measured on the text the tokenizer sees,
+# which is `serialize_state(state)` and not `str(state)`. On states a client can send the two
+# differ by up to 2x upward (a `"` costs one character in `repr` and two in JSON) and 10x downward
+# (`repr` escapes a zero-width space to six characters and a non-BMP format character to ten, where
+# `ensure_ascii=False` writes the one character it is). See `laya.serve._state_length`. Same
+# `getattr` shape as the bounds above.
+_state_length = getattr(_laya_serve, "_state_length", _fallback_state_length)
+
# The demo answers failures the way laya.serve does: a fixed message to the caller, the
# traceback to this logger. Without it a 500 arrived as a bare status line in the server
# output and the cause had to be reproduced in-process to be found at all.
@@ -268,9 +292,9 @@ def _check_request_limits(state: Any, questions: Dict[str, Any]) -> None:
Laya encodes the state once per question, so cost is questions x state size,
collated into one tensor, and a choice or score question adds one sequence per
- option against a shared head budget. The state length is measured exactly as
- laya.serve measures it -- `len(v)` for a string, `len(str(v))` for a dict or list
- -- and the option counts exactly as it counts them, over `choice` and `score`
+ option against a shared head budget. The state length comes from laya.serve's own
+ `_state_length`, so the two surfaces cannot disagree about what a state measures,
+ and the option counts exactly as it counts them, over `choice` and `score`
criteria only.
Checked here rather than declared as pydantic constraints on the request models,
@@ -284,7 +308,7 @@ def _check_request_limits(state: Any, questions: Dict[str, Any]) -> None:
status_code=413,
detail="too many questions (%d > %d)" % (len(questions), MAX_QUESTIONS),
)
- size = len(state) if isinstance(state, str) else len(str(state))
+ size = _state_length(state)
if size > MAX_STATE_CHARS:
raise HTTPException(
status_code=413,
@@ -3438,9 +3462,13 @@ async def gui_predict(request: Request) -> HTMLResponse:
try:
_check_request_limits(req.state, req.questions)
except HTTPException as exc:
- return _gui_error("Request too large", [str(exc.detail)],
- f"The server takes up to {MAX_QUESTIONS} questions and a state of up to "
- f"{MAX_STATE_CHARS:,} characters. Split the request or shorten the state.")
+ # Branch on the status: this check answers 400 as well as 413 now, and a size hint under
+ # the heading "Request too large" would describe the wrong problem.
+ if exc.status_code == 413:
+ return _gui_error("Request too large", [str(exc.detail)],
+ f"The server takes up to {MAX_QUESTIONS} questions and a state of up to "
+ f"{MAX_STATE_CHARS:,} characters. Split the request or shorten the state.")
+ return _gui_error("Bad request", [str(exc.detail)], "Fix the request and send it again.")
questions = _questions(req.questions)
try:
started = time.perf_counter()
diff --git a/laya/serve.py b/laya/serve.py
index ed08d39a..588ccd21 100644
--- a/laya/serve.py
+++ b/laya/serve.py
@@ -199,6 +199,52 @@ def _resolve_port() -> int:
return port
+def _state_length(state: Any) -> int:
+ r"""Length of the state text that will be tokenized; 400 if it has none.
+
+ `laya.common.serialize_state(state)` is what gets tokenized -- the state itself for a string,
+ `json.dumps(state, ensure_ascii=False)` for a dict or list, which the agents then strip mask
+ tokens out of, and that can only shorten it. `MAX_STATE_CHARS` has to be measured on that
+ text, not on `str()`, which is a different length in both directions:
+
+ * `repr` quotes a value with `'` and leaves a `"` inside it one character, where JSON escapes
+ it to the two characters `\"`. A `{"body": '"' * 49988}` state has a `len(str())` of exactly
+ 50000, so it passed the gate, and then serialized to 99988 -- the documented 50000-character
+ limit admitting very nearly twice what it says.
+ * `repr` renders a zero-width space as the six characters `\u200b` where `ensure_ascii=False`
+ writes the one character it is, so a state that serializes to 8344 characters -- a sixth of
+ the limit -- was refused with "state too large (50004 > 50000 chars)".
+
+ `json.dumps` inline rather than importing `serialize_state`: `laya.common` imports torch at
+ module level and `import laya.serve` must not (tests/test_lazy_import.py). The two are pinned
+ to each other in tests/test_serve.py.
+
+ The dump runs before the gate decides, so its bound is MAX_BODY_BYTES, not MAX_STATE_CHARS. At
+ the gate's own ceiling it is 0.133 ms against 0.080 ms for the `str()` it replaces, three
+ orders of magnitude under the forward pass that state then gets. On a 2 MiB body -- the largest
+ the streaming cap allows, and one this refuses -- it is 5.68 ms against 3.46 ms, on a request
+ that is rejected either way and whose `str()` upstream already walked in full. (Best of 200
+ after 20 warm-up iterations, CPython 3.12 on a 10-core arm64 laptop.) So the serialized text is
+ not worth threading through `predict()` to be encoded only once.
+ """
+ from fastapi import HTTPException
+
+ if isinstance(state, str):
+ return len(state) # `serialize_state` returns a string state unchanged
+ try:
+ return len(json.dumps(state, ensure_ascii=False))
+ except (TypeError, ValueError, RecursionError):
+ # A state `serialize_state` cannot render is not a size problem, and the 413 this replaces
+ # reported a count nothing had measured: `str()` raises on an integer of over 4300 digits
+ # (CPython's own int-to-str guard), and the old `except` answered "state too large (50001
+ # > 50000 chars)" for a state of a few kilobytes. No HTTP request reaches that case --
+ # `json.loads` builds only JSON types and refuses that integer itself, which
+ # `_systemone_inner` already answers 400 -- but in-process callers get here, and a state
+ # `json.dumps` refuses (a set, a datetime, a circular reference) used to pass the gate on
+ # its `str()` and then fail inside `serialize_state`, reported as a 500 "inference failed".
+ raise HTTPException(status_code=400, detail="'state' must be JSON-serializable")
+
+
def _check_request_limits(state: Any, questions: Any) -> None:
"""Reject absent or oversized inference requests before tokenization (400/413)."""
from fastapi import HTTPException
@@ -245,10 +291,7 @@ def _check_request_limits(state: Any, questions: Any) -> None:
detail="too many answer options across questions (%d > %d)" % (total_options, MAX_TOTAL_OPTIONS),
)
- try:
- state_len = len(state) if isinstance(state, str) else len(str(state))
- except Exception:
- state_len = MAX_STATE_CHARS + 1
+ state_len = _state_length(state)
if state_len > MAX_STATE_CHARS:
raise HTTPException(status_code=413,
detail="state too large (%d > %d chars)" % (state_len, MAX_STATE_CHARS))
diff --git a/tests/test_example_server_limits.py b/tests/test_example_server_limits.py
index 59781e6c..a1cc7845 100644
--- a/tests/test_example_server_limits.py
+++ b/tests/test_example_server_limits.py
@@ -261,11 +261,30 @@ def main():
+ [("n%d" % i, noul_crit) for i in range(30)])),
("noul only", "hi", {"n%d" % i: noul_crit for i in range(10)}),
("an ordinary request", {"body": "billed twice"}, one),
+ # A state whose `str()` and whose JSON serialization are different lengths (50000 vs 99988).
+ # This case catches the two surfaces DIVERGING -- one measuring the serialized text and the
+ # other `str(state)`. It cannot catch them being wrong together: `examples/server.py` binds
+ # `_state_length` from `laya.serve` by `getattr`, so a regression inside that helper moves
+ # both and parity stays green. Measured: regressing `_state_length` fails 0 of these parity
+ # checks and 4 of the absolute ones below. Those are the real coverage -- do not prune them
+ # as redundant.
+ ("a state whose repr is half its JSON", {"body": '"' * 49988}, one),
]
for label, st, qs in parity_cases:
s, d = serve_verdict(st, qs), demo_verdict(st, qs)
ok("parity with laya.serve: %s" % label, s == d, "serve=%s demo=%s" % (s, d))
+ # Parity alone cannot see a bug both surfaces share, and `getattr` guarantees they share one:
+ # with `_state_length` measuring `str(state)` again, both agree on accepting a state that
+ # serializes to 99 988 characters and every parity check above stays green (measured: 0 of them
+ # fail, 4 of these do). So the verdict itself is asserted, not just the agreement.
+ quote_heavy = {"body": '"' * 49988}
+ assert len(str(quote_heavy)) <= MAX_STATE_CHARS, "the fixture must pass a str()-based gate"
+ assert len(json.dumps(quote_heavy, ensure_ascii=False)) > MAX_STATE_CHARS, "and fail a real one"
+ for who, verdict in (("laya.serve", serve_verdict(quote_heavy, one)),
+ ("the demo server", demo_verdict(quote_heavy, one))):
+ ok("%s refuses a state whose JSON is twice its repr" % who, verdict == "refused", verdict)
+
# --- the resident-checkpoint cap: derived, not copied -------------------
# examples/server.py used to build its Router with `max_loaded=1`, a copy of a default
# laya/router.py retired in #180. A cap of one cannot hold both english and
diff --git a/tests/test_serve.py b/tests/test_serve.py
index dab066ab..1627fb1f 100644
--- a/tests/test_serve.py
+++ b/tests/test_serve.py
@@ -20,7 +20,9 @@ ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
from laya.serve import ( # noqa: E402
DEFAULT_MAX_TOKEN_BUDGET,
MAX_BODY_BYTES,
+ MAX_STATE_CHARS,
_apply_thread_limit,
+ _check_request_limits,
_env_bool,
_resolve_max_token_budget,
_resolve_max_loaded,
@@ -941,3 +943,134 @@ def test_resolve_max_token_budget_fallback(monkeypatch, caplog):
assert "invalid LAYA_MAX_TOKEN_BUDGET" in caplog.text
assert "LAYA_MAX_TOKEN_BUDGET must be positive" in caplog.text
+
+# --------------------------------------------------------------------------- #
+# The state-size gate has to measure the text that reaches the tokenizer
+# --------------------------------------------------------------------------- #
+
+# Two dict states whose `str()` and whose `serialize_state()` are different lengths, which is
+# what the gate measured wrongly. The numbers are exact rather than approximate: if either one
+# moves, the divergence these checks pin has moved with it.
+_QUOTE_STATE = {"body": '"' * 49988} # len(str()) == 50000, serialized == 99988
+_ZWSP_STATE = {"body": "\u200b" * 8332} # len(str()) == 50004, serialized == 8344
+
+
+def _serialized_len(state):
+ """The length `laya.common.serialize_state` produces, without importing torch."""
+ return len(json.dumps(state, ensure_ascii=False))
+
+
+def test_the_gate_counts_the_json_the_tokenizer_will_see(monkeypatch):
+ """`repr` leaves a `"` inside a value one character; JSON escapes it to two.
+
+ So a state sitting exactly on the 50000-character gate was tokenized as 99988 characters --
+ the documented limit admitting very nearly twice what it says.
+ """
+ assert len(str(_QUOTE_STATE)) == MAX_STATE_CHARS # exactly at a repr-measured gate
+ assert _serialized_len(_QUOTE_STATE) == 99988 # what build_sequence would encode
+ client, fake = _client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "state": _QUOTE_STATE})
+ assert r.status_code == 413
+ assert r.json()["detail"] == "state too large (99988 > %d chars)" % MAX_STATE_CHARS
+ assert fake.calls == [] # refused before inference, as intended
+
+
+def test_a_state_a_sixth_of_the_limit_is_not_refused_for_its_repr(monkeypatch):
+ """The same mismatch the other way: `repr` escapes a zero-width space to six characters.
+
+ `ensure_ascii=False` writes the one character it is, so this state serializes to 8344 --
+ and was answered "state too large (50004 > 50000 chars)".
+ """
+ assert len(str(_ZWSP_STATE)) == 50004 # over a repr-measured gate
+ assert _serialized_len(_ZWSP_STATE) == 8344 # a sixth of the limit, once serialized
+ client, fake = _client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "state": _ZWSP_STATE})
+ assert r.status_code == 200
+ assert fake.calls[0]["state"] == _ZWSP_STATE # reached inference unchanged
+
+
+# Deliberately states whose `repr` and whose JSON are the same length, so these four answer the
+# same before and after the fix. They are here to pin what must NOT move: the limit still admits a
+# state that sits exactly on it and still refuses one character more.
[email protected]("state, expected", [
+ ({"body": "a" * 49988}, 200), # str() and JSON are both exactly 50000
+ ({"body": "a" * 49989}, 413), # both exactly one character over
+ ("A" * MAX_STATE_CHARS, 200), # a string state is its own serialization
+ ("A" * (MAX_STATE_CHARS + 1), 413),
+])
+def test_the_declared_limit_is_the_limit_for_ordinary_states(monkeypatch, state, expected):
+ client, _ = _client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "state": state})
+ assert r.status_code == expected
+
+
+def test_the_gate_agrees_with_the_function_that_feeds_the_tokenizer():
+ """Pin serve.py's inline `json.dumps` to `serialize_state` itself.
+
+ serve.py cannot import it -- `laya.common` imports torch at module level and
+ `import laya.serve` must not (tests/test_lazy_import.py) -- so the mirror is held here
+ instead. Looked up rather than imported at module scope so that a serve.py without the
+ helper fails this one check instead of aborting the suite's import.
+ """
+ import laya.serve as serve_mod
+
+ state_length = getattr(serve_mod, "_state_length", None)
+ assert state_length is not None, "laya.serve no longer measures the state through _state_length"
+ serialize_state = pytest.importorskip("laya.common").serialize_state
+ for state in (_QUOTE_STATE, _ZWSP_STATE, "plain text", {}, [],
+ {"a": {"b": ["c", 1, 1.5, None, True]}},
+ ["\u4e2d\u6587", '"', "'", "\n", "\x00", "\U0001f600"],
+ # A non-BMP format character is the widest divergence a client can send: `repr`
+ # writes it as the ten characters `\U000e0001`, `ensure_ascii=False` as one.
+ {"tag": "\U000e0001" * 10},
+ {"d": "it's", "q": 'say "hi"'}):
+ assert state_length(state) == len(serialize_state(state)), state
+
+
+def default_int_digits():
+ """Pin CPython's int-to-str digit guard for the tests that rely on it.
+
+ The guard defaults to 4300 but is operator-configurable (`PYTHONINTMAXSTRDIGITS`,
+ `sys.set_int_max_str_digits`). With it switched off, `json.dumps(10 ** 4300)` succeeds and
+ `json.loads` accepts a 4301-digit body, so these two tests fail while the code under test stays
+ correct -- verified: `PYTHONINTMAXSTRDIGITS=0` turns both of them red on an unmodified tree. A
+ test that depends on an operator's setting is measuring the environment, not the change.
+ """
+ previous = sys.get_int_max_str_digits()
+ sys.set_int_max_str_digits(4300)
+ try:
+ yield
+ finally:
+ sys.set_int_max_str_digits(previous)
+
+
+def test_an_unserializable_state_is_a_400_not_a_count_nothing_measured(default_int_digits):
+ """The old `except Exception` answered 413 "state too large (50001 > 50000 chars)".
+
+ That reported a length nothing had measured, for a state that may be a few bytes. These
+ states are unreachable over HTTP -- `json.loads` builds only JSON types -- but they are
+ what an in-process caller of this helper can pass, and on the old path a state
+ `serialize_state` cannot render passed the gate and failed later as a 500.
+ """
+ from fastapi import HTTPException
+
+ # 10 ** 4300 has 4301 digits, one past CPython's int-to-str limit, and is built by
+ # arithmetic because `int("1" * 4301)` would hit that same limit while parsing.
+ for state in ({1, 2}, 10 ** 4300, {"n": 10 ** 4300}, [10 ** 4300]):
+ with pytest.raises(HTTPException) as exc:
+ _check_request_limits(state, REQ["questions"])
+ assert exc.value.status_code == 400
+ # The whole detail, so a count nothing measured cannot creep back into it.
+ assert exc.value.detail == "'state' must be JSON-serializable"
+
+
+def test_an_oversized_integer_state_never_reaches_the_gate(monkeypatch, default_int_digits):
+ """Why the 400 above is not an HTTP-reachable case: the parser refuses the body first."""
+ client, fake = _client(monkeypatch)
+ r = client.post("/v1/systemone",
+ content=b'{"state": ' + b"1" * 4301 + b', "questions": {}}',
+ headers={"content-type": "application/json"})
+ assert r.status_code == 400
+ assert r.json()["detail"] == "request body must be valid JSON"
+ assert fake.calls == []