Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #724 from aashish254/fix-serve-per-call-controls

Nandakishor66f2ea53268a
diff --cc README.md
index 1bc815a7,fdd4e731..328a6194
--- a/README.md
+++ b/README.md
@@@ -567,12 -498,12 +567,16 @@@ reachable on demand, or the server rebu
and `LAYA_API_KEY` (when set, clients must
send `Authorization: Bearer <key>`). A client's `model` field is honoured when it
names a Laya checkpoint (`english`/`multilingual`/`typed-decisions`), otherwise
- the router auto-selects by script/language.
+ the router auto-selects by script/language. The same body can also carry `task`,
+ `lang`, `lang_guess`, `min_confidence`, `max_len` and `head_max_len` — the controls
+ `Router.predict` takes that a JSON body can state — while the five hook arguments
+ (`hooks`, `on_predict_start`, `on_predict_end`, `hooks_raise`, `hooks_timeout`) are refused
+ with 422, because a hook is a callable that has to live where the server runs.
+When publishing the server under a reverse-proxy prefix such as `/laya`, set
+`LAYA_ROOT_PATH=/laya`. The proxy should strip that prefix before forwarding;
+the setting controls FastAPI-generated URLs and leaves the internal routes unchanged.
+
Three things differ from Jev when you port a client:
* **Options per question.** A question's options share the checkpoint's option budget, `head_max_len` (192 tokens on `laya`, 256 on the other two), not Jev's cap of 255 options. In addition, the HTTP server (`laya.serve`) enforces an amplification guard of at most 100 choice options per question (`MAX_CHOICE_OPTIONS = 100`, rejected with 413 before inference). Once options overflow the token budget, around 20 options with a short description each, every option is trimmed to fit, so long or similar labels can reach the model reading the same ([Where Jev leads](#where-jev-leads)). Once they no longer fit the window at all, the library rejects the request with 422. For more candidates, narrow them first with `predict_shortlist` ([Honest limits](#honest-limits)).
diff --cc laya/serve.py
index f546aed4,6704d1f9..59a30753
--- a/laya/serve.py
+++ b/laya/serve.py
@@@ -76,7 -78,25 +76,18 @@@ MAX_TOTAL_OPTIONS = 51
DEFAULT_MAX_CONCURRENT = 16
# Server-side ceiling on per-request max_len/head_max_len token budget overrides.
DEFAULT_MAX_TOKEN_BUDGET = 8192
-# Public Hugging Face ids, accepted so a client can name a checkpoint. The root bundle is
-# deliberately absent: the documented ``convaiinnovations/laya`` value means
-# "let the Router choose", rather than pinning the English checkpoint.
-_PUBLISHED_MODEL_IDS = {
- "convaiinnovations/laya-multilingual": "multilingual",
- "convaiinnovations/laya-typed-decisions": "typed-decisions",
-}
+ # Every control ``Router.predict`` takes beyond ``state`` and ``questions``, split by whether a
+ # JSON body can mean it. ``tests/test_serve.py`` pins the two lists against
+ # ``inspect.signature(Router.predict)``, so a control added to the core call has to be placed on
+ # one side of that line -- forwarded or refused -- before the suite goes green again. The line is
+ # not drawn by value type: a hook is a callable that runs in the server process, and
+ # ``hooks_raise`` / ``hooks_timeout`` govern how the hooks the *operator* installed execute, so
+ # none of those five has a meaning a request could give it. Which checkpoint answers, which
+ # language it reads, how many tokens it gets and where it abstains are all plain data.
+ BODY_CONTROLS = ("model", "max_len", "head_max_len", "task", "lang", "lang_guess", "min_confidence")
+ BODY_REFUSALS = ("hooks", "on_predict_start", "on_predict_end", "hooks_raise", "hooks_timeout")
+
def _env_bool(name: str, default: bool) -> bool:
v = os.environ.get(name)
@@@ -636,16 -536,7 +706,17 @@@ def create_app(router: Optional[Any] =
state = body.get("state")
questions = body["questions"]
_check_request_limits(state, questions)
+ _refuse_body_refusals(body)
+ # After the size checks, so an oversized body is refused before anything walks it, and
+ # `MAX_STATE_CHARS`/`MAX_QUESTIONS` bound what the walk can reach. A `\udXXX` escape with
+ # no pair is legal JSON that cannot be UTF-8 encoded, so the tokenizer raised `TypeError`
+ # from inside `build_sequence` and the client's own mistake came back as a 500
+ # "inference failed". A *paired* surrogate is an ordinary astral character (an emoji) by
+ # the time `json.loads` is done, so only lone ones are rejected here.
+ if _has_lone_surrogate(body):
+ raise HTTPException(status_code=400,
+ detail="request body contains an unpaired surrogate escape; "
+ "those cannot be encoded as UTF-8")
model = _resolve_model(body.get("model"))
max_budget_cap = _resolve_max_token_budget()
max_len = _validate_budget_param(body, "max_len", max_budget_cap)
diff --cc tests/test_serve.py
index 35f89395,6c9c6d77..e41dc9f8
--- a/tests/test_serve.py
+++ b/tests/test_serve.py
@@@ -18,11 -19,11 +19,13 @@@ from fastapi.testclient import TestClie
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
from laya.serve import ( # noqa: E402
+ BODY_CONTROLS,
+ BODY_REFUSALS,
DEFAULT_MAX_TOKEN_BUDGET,
MAX_BODY_BYTES,
+ MAX_STATE_CHARS,
_apply_thread_limit,
+ _check_request_limits,
_env_bool,
_resolve_max_token_budget,
_resolve_max_loaded,
@@@ -1039,502 -944,209 +1042,708 @@@ def test_resolve_max_token_budget_fallb
assert "invalid LAYA_MAX_TOKEN_BUDGET" in caplog.text
assert "LAYA_MAX_TOKEN_BUDGET must be positive" in caplog.text
+BATCH_REQ = {
+ "states": ["first state", "second state"],
+ "questions": REQ["questions"],
+}
+
+
+def test_batch_happy_path(monkeypatch):
+ client, fake = _client(monkeypatch)
+ r = client.post("/v1/systemone/batch", json=BATCH_REQ)
+ assert r.status_code == 200, r.text
+ data = r.json()
+ assert "results" in data
+ assert len(data["results"]) == 2
+ assert "total_usage" in data
+ assert data["total_usage"]["input_tokens"] == sum(
+ res["usage"]["input_tokens"] for res in data["results"]
+ )
+ assert len(fake.calls) == 2
+ assert "Server-Timing" in r.headers
+ assert "X-Inference-Time-Ms" in r.headers
+
+
+def test_batch_missing_state_in_list_returns_400(monkeypatch):
+ """A None state inside states list must be rejected with 400 'state' is required."""
+ client, _ = _client(monkeypatch)
+ bad_req = {"states": [None, "hello"], "questions": REQ["questions"]}
+ r = client.post("/v1/systemone/batch", json=bad_req)
+ assert r.status_code == 400
+ assert "'state' is required" in r.json()["detail"]
+
+
+def test_batch_too_many_options_returns_413(monkeypatch):
+ """Questions with more than MAX_CHOICE_OPTIONS must be rejected with 413."""
+ client, _ = _client(monkeypatch)
+ bad_questions = {
+ "dept": {
+ "type": "choice",
+ "instructions": "which?",
+ "criteria": {f"opt_{i}": f"desc_{i}" for i in range(101)},
+ }
+ }
+ r = client.post("/v1/systemone/batch", json={"states": ["state"], "questions": bad_questions})
+ assert r.status_code == 413
+ assert "too many choice options" in r.json()["detail"]
+
+
+def test_batch_empty_states_returns_400(monkeypatch):
+ client, _ = _client(monkeypatch)
+ r = client.post("/v1/systemone/batch", json={"states": [], "questions": REQ["questions"]})
+ assert r.status_code == 400
+ assert "states" in r.json()["detail"]
+
+
+def test_batch_missing_states_returns_400(monkeypatch):
+ client, _ = _client(monkeypatch)
+ r = client.post("/v1/systemone/batch", json={"questions": REQ["questions"]})
+ assert r.status_code == 400
+
+
+def test_batch_missing_questions_returns_400(monkeypatch):
+ client, _ = _client(monkeypatch)
+ r = client.post("/v1/systemone/batch", json={"states": ["state 1"]})
+ assert r.status_code == 400
+
+
+def test_batch_too_many_states_returns_413(monkeypatch):
+ client, _ = _client(monkeypatch)
+ from laya.serve import MAX_BATCH_STATES
+ oversized = {"states": ["state"] * (MAX_BATCH_STATES + 1), "questions": REQ["questions"]}
+ r = client.post("/v1/systemone/batch", json=oversized)
+ assert r.status_code == 413
+ assert "too many states" in r.json()["detail"]
+
+
+def test_batch_individual_oversized_state_returns_413(monkeypatch):
+ client, _ = _client(monkeypatch)
+ from laya.serve import MAX_STATE_CHARS
+ bad_req = {
+ "states": ["ok", "X" * (MAX_STATE_CHARS + 10)],
+ "questions": REQ["questions"],
+ }
+ r = client.post("/v1/systemone/batch", json=bad_req)
+ assert r.status_code == 413
+ assert "state too large" in r.json()["detail"]
+
+
+def test_batch_auth_required_when_key_set(monkeypatch):
+ client, _ = _client(monkeypatch, api_key="secret123")
+ r = client.post("/v1/systemone/batch", json=BATCH_REQ)
+ assert r.status_code == 401
+ ok = client.post(
+ "/v1/systemone/batch",
+ json=BATCH_REQ,
+ headers={"Authorization": "Bearer secret123"},
+ )
+ assert ok.status_code == 200
+
+
+def test_batch_admission_bound_refuses_with_503(monkeypatch):
+ """Batch endpoint must respect the admission bound and return 503 when busy."""
+ import asyncio
+ import httpx
+
+ monkeypatch.delenv("LAYA_API_KEY", raising=False)
+ monkeypatch.setenv("LAYA_MAX_CONCURRENT", "1")
+ fake = GatedRouter()
+ app = create_app(router=fake)
+ seen = {}
+
+ async def drive():
+ transport = httpx.ASGITransport(app=app)
+ async with httpx.AsyncClient(transport=transport, base_url="http://t") as client:
+ first = asyncio.ensure_future(client.post("/v1/systemone", json=REQ))
+ for _ in range(200):
+ if fake.entered.is_set():
+ break
+ await asyncio.sleep(0.05)
+ assert fake.entered.is_set(), "first request never reached inference"
+ await asyncio.sleep(0.2)
+ seen["batch_second"] = (await client.post("/v1/systemone/batch", json=BATCH_REQ)).status_code
+ fake.release.set()
+ seen["first"] = (await first).status_code
+
+ asyncio.run(drive())
+
+ assert seen["batch_second"] == 503, seen
+ assert seen["first"] == 200, seen
+
+
+class BatchCapableFakeRouter(FakeRouter):
+ """FakeRouter that implements predict_batch."""
+
+ def __init__(self):
+ super().__init__()
+ self.batch_calls = []
+
+ def predict_batch(self, requests, batch_size=None):
+ self.batch_calls.append(requests)
+ return [
+ {
+ "model": "laya-rl-agent",
+ "answers": {
+ "dept": {
+ "type": "choice",
+ "choice": "billing",
+ "probabilities": {"billing": 0.94, "tech": 0.06},
+ "confidence": 0.94,
+ }
+ },
+ "usage": {"input_tokens": 42, "output_tokens": 0},
+ "routing": {"model": "english", "reason": "English Latin text"},
+ }
+ for _ in requests
+ ]
+
+
+def test_batch_uses_predict_batch_when_available(monkeypatch):
+ monkeypatch.delenv("LAYA_API_KEY", raising=False)
+ fake = BatchCapableFakeRouter()
+ client = TestClient(create_app(router=fake))
+ r = client.post("/v1/systemone/batch", json=BATCH_REQ)
+ assert r.status_code == 200, r.text
+ assert len(fake.batch_calls) == 1
+ assert len(fake.batch_calls[0]) == 2
+ assert len(fake.calls) == 0 # Confirms no sequential predict() fallback calls were made
+def test_an_unpaired_surrogate_is_a_caller_error_not_a_server_fault():
+ r"""A `\udXXX` escape with no pair parses as JSON and then cannot be UTF-8 encoded.
+
+ The tokenizer raised `TypeError` from inside `build_sequence`, and `serve` maps only
+ `ValueError` to 422, so a malformed string in the client's own body came back as a 500
+ "inference failed" -- a server fault plus an operator traceback for the caller's mistake.
+ A *paired* surrogate is an ordinary astral character (an emoji) and must keep working.
+ """
+ seen = []
+
+ class RecordingRouter:
+ loaded = ["english"]
+
+ def predict(self, state, questions, model=None):
+ seen.append(state)
+ return {"model": "stub", "answers": {}, "usage": {}}
+
+ client = TestClient(create_app(router=RecordingRouter()), raise_server_exceptions=False)
+
+ lone = [
+ b'{"state":"\\ud800","questions":{"q":{"type":"noul","instructions":"x"}}}',
+ b'{"state":"ok","questions":{"q":{"type":"noul","instructions":"\\udfff"}}}',
+ b'{"state":"ok","questions":{"q":{"type":"choice","instructions":"x",'
+ b'"criteria":["\\ud800","b"]}}}',
+ ]
+ for body in lone:
+ res = client.post("/v1/systemone", content=body,
+ headers={"content-type": "application/json"})
+ assert res.status_code == 400, (body, res.status_code, res.text)
+ assert "unpaired surrogate" in res.text, res.text
+
+ # a paired surrogate is one astral character by the time json.loads is done: it must reach
+ # the router rather than be rejected
+ seen.clear()
+ ok = json.dumps({"state": "hi \U0001f600",
+ "questions": {"q": {"type": "noul", "instructions": "x"}}}).encode("utf-8")
+ res = client.post("/v1/systemone", content=ok,
+ headers={"content-type": "application/json"})
+ assert res.status_code == 200, (res.status_code, res.text)
+ assert seen and "\U0001f600" in str(seen[0]), seen
+
+
+def test_a_deeply_nested_state_is_not_a_recursion_error():
+ """`json.loads` accepts nesting far deeper than Python's recursion limit.
+
+ A recursive walk over the parsed body therefore turned a body the parser handles into a
+ `RecursionError`, i.e. one 500 replaced by another. The walk uses an explicit stack, so the
+ depth the JSON parser accepts is the depth this handles.
+ """
+ class NestedRouter:
+ loaded = ["english"]
+
+ def predict(self, state, questions, model=None):
+ return {"model": "stub", "answers": {}, "usage": {}}
+
+ client = TestClient(create_app(router=NestedRouter()), raise_server_exceptions=False)
+ questions = {"q": {"type": "noul", "instructions": "x"}}
+
+ for depth in (200, 600, 1200):
+ body = json.dumps({"state": ["x" * 3] * depth, "questions": questions}).encode("utf-8")
+ res = client.post("/v1/systemone", content=body,
+ headers={"content-type": "application/json"})
+ assert res.status_code == 200, (depth, res.status_code, res.text)
+def test_inference_timing_headers_absent_on_error():
+ """Inference timing headers must not be attached to error responses."""
+ router = FakeRouter()
+ client = TestClient(create_app(router=router))
+ res = client.post("/v1/systemone", json={"state": "missing questions"})
+ assert res.status_code == 400
+ assert "Server-Timing" not in res.headers
+ assert "X-Inference-Time-Ms" not in res.headers
+
+
+# --------------------------------------------------------------------------- #
+# The state-size gate has to measure the text that reaches the tokenizer
+# --------------------------------------------------------------------------- #
+
+# Two dict states whose `str()` and whose `serialize_state()` are different lengths, which is
+# what the gate measured wrongly. The numbers are exact rather than approximate: if either one
+# moves, the divergence these checks pin has moved with it.
+_QUOTE_STATE = {"body": '"' * 49988} # len(str()) == 50000, serialized == 99988
+_ZWSP_STATE = {"body": "\u200b" * 8332} # len(str()) == 50004, serialized == 8344
+
+
+def _serialized_len(state):
+ """The length `laya.common.serialize_state` produces, without importing torch."""
+ return len(json.dumps(state, ensure_ascii=False))
+
+
+def test_the_gate_counts_the_json_the_tokenizer_will_see(monkeypatch):
+ """`repr` leaves a `"` inside a value one character; JSON escapes it to two.
+
+ So a state sitting exactly on the 50000-character gate was tokenized as 99988 characters --
+ the documented limit admitting very nearly twice what it says.
+ """
+ assert len(str(_QUOTE_STATE)) == MAX_STATE_CHARS # exactly at a repr-measured gate
+ assert _serialized_len(_QUOTE_STATE) == 99988 # what build_sequence would encode
+ client, fake = _client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "state": _QUOTE_STATE})
+ assert r.status_code == 413
+ assert r.json()["detail"] == "state too large (99988 > %d chars)" % MAX_STATE_CHARS
+ assert fake.calls == [] # refused before inference, as intended
+
+
+def test_a_state_a_sixth_of_the_limit_is_not_refused_for_its_repr(monkeypatch):
+ """The same mismatch the other way: `repr` escapes a zero-width space to six characters.
+
+ `ensure_ascii=False` writes the one character it is, so this state serializes to 8344 --
+ and was answered "state too large (50004 > 50000 chars)".
+ """
+ assert len(str(_ZWSP_STATE)) == 50004 # over a repr-measured gate
+ assert _serialized_len(_ZWSP_STATE) == 8344 # a sixth of the limit, once serialized
+ client, fake = _client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "state": _ZWSP_STATE})
+ assert r.status_code == 200
+ assert fake.calls[0]["state"] == _ZWSP_STATE # reached inference unchanged
+
+
+# Deliberately states whose `repr` and whose JSON are the same length, so these four answer the
+# same before and after the fix. They are here to pin what must NOT move: the limit still admits a
+# state that sits exactly on it and still refuses one character more.
+# The explicit ids are required, not cosmetic. pytest builds each case's node id out of the
+# parameter value, and writes that node id into the PYTEST_CURRENT_TEST environment variable -- a
+# 50 000-character state produces a 50 084-character id, and a Windows environment variable cannot
+# exceed 32 767 characters, so every case errors at teardown with
+# `ValueError: the environment variable is longer than 32767 characters`. It passes on Linux, which
+# has no comparable per-variable limit. `b"[" * 100000` in this file carries an id for the same
+# reason.
[email protected]("state, expected", [
+ # str() and JSON are both exactly 50000
+ pytest.param({"body": "a" * 49988}, 200, id="dict-exactly-at-the-limit"),
+ # both exactly one character over
+ pytest.param({"body": "a" * 49989}, 413, id="dict-one-character-over"),
+ # a string state is its own serialization
+ pytest.param("A" * MAX_STATE_CHARS, 200, id="str-exactly-at-the-limit"),
+ pytest.param("A" * (MAX_STATE_CHARS + 1), 413, id="str-one-character-over"),
+])
+def test_the_declared_limit_is_the_limit_for_ordinary_states(monkeypatch, request, state, expected):
+ # Asserted here rather than left to the Windows job: pytest writes this node id into
+ # PYTEST_CURRENT_TEST, where Windows caps a variable at 32 767 characters, so dropping the
+ # explicit ids above turns every case in this test into a teardown error -- on Windows only.
+ # This assertion fails on any platform, so the ids cannot be lost without CI saying so.
+ assert len(request.node.nodeid) < 32767, \
+ "node id is %d characters; give this parameter an explicit short id" % len(request.node.nodeid)
+ client, _ = _client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "state": state})
+ assert r.status_code == expected
+
+
+def test_the_gate_agrees_with_the_function_that_feeds_the_tokenizer():
+ """Pin serve.py's inline `json.dumps` to `serialize_state` itself.
+
+ serve.py cannot import it -- `laya.common` imports torch at module level and
+ `import laya.serve` must not (tests/test_lazy_import.py) -- so the mirror is held here
+ instead. Looked up rather than imported at module scope so that a serve.py without the
+ helper fails this one check instead of aborting the suite's import.
+ """
+ import laya.serve as serve_mod
+
+ state_length = getattr(serve_mod, "_state_length", None)
+ assert state_length is not None, "laya.serve no longer measures the state through _state_length"
+ serialize_state = pytest.importorskip("laya.common").serialize_state
+ for state in (_QUOTE_STATE, _ZWSP_STATE, "plain text", {}, [],
+ {"a": {"b": ["c", 1, 1.5, None, True]}},
+ ["\u4e2d\u6587", '"', "'", "\n", "\x00", "\U0001f600"],
+ # A non-BMP format character is the widest divergence a client can send: `repr`
+ # writes it as the ten characters `\U000e0001`, `ensure_ascii=False` as one.
+ {"tag": "\U000e0001" * 10},
+ {"d": "it's", "q": 'say "hi"'}):
+ assert state_length(state) == len(serialize_state(state)), state
+
+
+def default_int_digits():
+ """Pin CPython's int-to-str digit guard for the tests that rely on it.
+
+ The guard defaults to 4300 but is operator-configurable (`PYTHONINTMAXSTRDIGITS`,
+ `sys.set_int_max_str_digits`). With it switched off, `json.dumps(10 ** 4300)` succeeds and
+ `json.loads` accepts a 4301-digit body, so these two tests fail while the code under test stays
+ correct -- verified: `PYTHONINTMAXSTRDIGITS=0` turns both of them red on an unmodified tree. A
+ test that depends on an operator's setting is measuring the environment, not the change.
+ """
+ previous = sys.get_int_max_str_digits()
+ sys.set_int_max_str_digits(4300)
+ try:
+ yield
+ finally:
+ sys.set_int_max_str_digits(previous)
+
+
+def test_an_unserializable_state_is_a_400_not_a_count_nothing_measured(default_int_digits):
+ """The old `except Exception` answered 413 "state too large (50001 > 50000 chars)".
+
+ That reported a length nothing had measured, for a state that may be a few bytes. These
+ states are unreachable over HTTP -- `json.loads` builds only JSON types -- but they are
+ what an in-process caller of this helper can pass, and on the old path a state
+ `serialize_state` cannot render passed the gate and failed later as a 500.
+ """
+ from fastapi import HTTPException
+
+ # 10 ** 4300 has 4301 digits, one past CPython's int-to-str limit, and is built by
+ # arithmetic because `int("1" * 4301)` would hit that same limit while parsing.
+ for state in ({1, 2}, 10 ** 4300, {"n": 10 ** 4300}, [10 ** 4300]):
+ with pytest.raises(HTTPException) as exc:
+ _check_request_limits(state, REQ["questions"])
+ assert exc.value.status_code == 400
+ # The whole detail, so a count nothing measured cannot creep back into it.
+ assert exc.value.detail == "'state' must be JSON-serializable"
+
+
+def test_an_oversized_integer_state_never_reaches_the_gate(monkeypatch, default_int_digits):
+ """Why the 400 above is not an HTTP-reachable case: the parser refuses the body first."""
+ client, fake = _client(monkeypatch)
+ r = client.post("/v1/systemone",
+ content=b'{"state": ' + b"1" * 4301 + b', "questions": {}}',
+ headers={"content-type": "application/json"})
+ assert r.status_code == 400
+ assert r.json()["detail"] == "request body must be valid JSON"
+ assert fake.calls == []
+
+
+# --- the lower-bound probe in front of the exact measurement ---------------------------------
+
+def test_the_probe_refuses_an_oversized_value_without_serializing_the_state(monkeypatch):
+ """An oversized state is refused without the 2 MiB dump whose result cannot change the verdict.
+
+ This is the check that goes red if `_state_length_lower_bound_over` is removed: without it the gate
+ still answers 413, so the *absence of the dump* is the only observable difference.
+ """
+ import laya.serve as serve_mod
+
+ calls = []
+ real_dumps = json.dumps
+
+ def counting_dumps(obj, **kw):
+ calls.append(kw)
+ return real_dumps(obj, **kw)
+
+ # Scoped to the module's own binding: `serve_mod.json` IS the stdlib module, so patching an
+ # attribute on it would replace `json.dumps` for the whole process for the test's duration.
+ monkeypatch.setattr(serve_mod, "json", SimpleNamespace(dumps=counting_dumps))
+ # F2: a conversation LIST is this library's dominant state shape, and the probe's list branch
+ # can be deleted with every other check still green unless one of these is list-shaped.
+ for state in ({"body": "a" * (2 * 1024 * 1024)},
+ ["z" * 60000],
+ [{"role": "user", "content": "x" * 70000}],
+ ({"role": "user", "content": "x" * 70000},)):
+ calls.clear()
+ assert serve_mod._state_length(state) > serve_mod.MAX_STATE_CHARS, state.__class__.__name__
+ assert calls == [], "a %s state was serialized although one value already exceeds the cap" \
+ % type(state).__name__
+
+
+def test_the_probe_verdict_always_matches_the_exact_measurement():
+ """The probe may only report "provably over"; its verdict must never differ from the encoder's.
+
+ An unsound probe would be a security bug in the same family as the one this PR fixes, so the two
+ are compared across shapes placed deliberately on both sides of the limit -- including the
+ escape-heavy values where `repr` and JSON diverge, which is what the gate exists for.
+ """
+ import laya.serve as serve_mod
+ cap = serve_mod.MAX_STATE_CHARS
+ shapes = []
+ for ch in ("a", '"', "\u200b", "\U000e0001", "\n", "\\", "\u4e2d"):
+ for n in (1, 100, cap // 2, cap - 1, cap, cap + 1, cap * 2):
+ shapes.append({"body": ch * n})
+ shapes.append([ch * n])
+ shapes.append({"a": {"b": [ch * n]}})
+ shapes += [
+ {("k%05d" % i): "v" for i in range(3000)},
+ {("k%05d" % i): "v" * 40 for i in range(3000)},
+ [{"role": "user", "content": "hi"} for _ in range(4000)],
+ {"n": 1, "f": 1.5, "t": True, "z": None},
+ {},
+ [],
+ ]
+ assert len(shapes) == 7 * 7 * 3 + 6, len(shapes)
+ for state in shapes:
+ exact = len(json.dumps(state, ensure_ascii=False))
+ probed = serve_mod._state_length(state)
+ assert (probed > cap) == (exact > cap), \
+ "probe and encoder disagree for a %d-character state" % exact
+ if exact <= cap:
+ # Stronger than agreeing on the verdict: under the cap the *exact* length is returned,
+ # so the 413 message keeps quoting a number something actually measured.
+ assert probed == exact
+
+
+def test_the_early_refusal_reports_a_number_it_measured():
+ """The 413 must not invent a count. Reporting `cap + 1` would answer "50001 > 50000" for a
+ 60 012-character state -- the same fabricated count this gate was written to remove.
+ """
+ import laya.serve as serve_mod
+ cap = serve_mod.MAX_STATE_CHARS
+ # `cap + 1` is NOT the tell: {"body": "a" * 50001} legitimately sums to exactly that. The
+ # property that distinguishes a measured bound from a fabricated one is `cap < reported <= exact`.
+ for state in ({"body": "x" * 60000}, {"body": "y" * (2 * 1024 * 1024)}, {"a": ["z" * 80000]},
+ {"body": "a" * (cap + 1)}, ["w" * 60000]):
+ exact = len(json.dumps(state, ensure_ascii=False))
+ reported = serve_mod._state_length(state)
+ assert reported > cap, reported
+ assert reported <= exact, \
+ "reported %d for a state of %d characters -- a 413 must never overstate" % (reported, exact)
+
+
+def test_the_probe_is_bounded_and_falls_through_rather_than_guessing():
+ """The probe examines a constant number of values, so it can never become the expensive step.
+
+ A state of 200 000 small values is not provably over within that budget, and must fall through
+ to the encoder rather than be refused on a partial sum.
+ """
+ import laya.serve as serve_mod
+ assert serve_mod._STATE_PROBE_VALUES <= 256
+ wide = {("k%06d" % i): "x" for i in range(200000)}
+ assert serve_mod._state_length_lower_bound_over(wide, serve_mod.MAX_STATE_CHARS) == 0
+ assert serve_mod._state_length(wide) == len(json.dumps(wide, ensure_ascii=False))
+
+
+def test_the_probe_walks_exact_container_types_only():
+ """A subclass may override `values()` while `json.dumps` reads the real items, so walking one
+ would let the "lower bound" exceed the true length. Measured before this was narrowed: a
+ 13-character state refused as `60000 > 50000`. No HTTP request can reach it -- `json.loads`
+ builds exact types -- but the 400 branch exists for in-process callers.
+ """
+ import laya.serve as serve_mod
+
+ class Lying(dict):
+ def values(self):
+ return ["x" * 60000]
+
+ state = Lying(x="tiny")
+ exact = len(json.dumps(state, ensure_ascii=False))
+ assert serve_mod._state_length_lower_bound_over(state, serve_mod.MAX_STATE_CHARS) == 0
+ assert serve_mod._state_length(state) == exact == 13
+
+ # --------------------------------------------------------------------- per-call controls
+ #
+ # `Router.predict` takes nine arguments beyond `state` and `questions` that a JSON body could
+ # state. `laya/serve.py` splits them into the two lists the tests below read, and the first test
+ # pins those lists against core's own signature: a control added to `predict` cannot be silently
+ # ignored by the HTTP surface, because it has to be placed on one side of that line first.
+
+ NEW_CONTROLS = ("task", "lang", "lang_guess", "min_confidence")
+ NEW_CONTROL_VALUES = {"task": "typed-decisions", "lang": "de", "lang_guess": "de",
+ "min_confidence": 0.9}
+
+
+ def test_every_predict_control_is_forwarded_or_refused():
+ """`BODY_CONTROLS` and `BODY_REFUSALS` must cover `Router.predict` exactly, both directions."""
+ from laya.router import Router
+
+ taken = set(inspect.signature(Router.predict).parameters) - {"self", "state", "questions"}
+ declared = set(BODY_CONTROLS) | set(BODY_REFUSALS)
+ assert not set(BODY_CONTROLS) & set(BODY_REFUSALS)
+ assert taken == declared, "predict() takes %s; serve declares %s" % (
+ sorted(taken), sorted(declared))
+
+
+ @pytest.mark.parametrize("key", NEW_CONTROLS)
+ def test_each_control_reaches_predict(monkeypatch, key):
+ client, fake = _budget_client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, key: NEW_CONTROL_VALUES[key]})
+ assert r.status_code == 200, r.text
+ assert fake.calls[0][key] == NEW_CONTROL_VALUES[key]
+
+
+ def test_unset_controls_are_not_sent_as_none(monkeypatch):
+ """An absent control must stay absent rather than be forwarded as `None`.
+
+ Core reads `None` as "inherit what the Router was built with", so sending `lang_guess=None`
+ would override a deployment's `Router(lang_guess=...)`. It would also 500 on any router whose
+ `predict` does not take the keyword -- the `FakeRouter` this suite has always used.
+ """
+ client, fake = _budget_client(monkeypatch)
+ assert client.post("/v1/systemone", json=REQ).status_code == 200
+ assert not [key for key in NEW_CONTROLS if key in fake.calls[0]]
+
+
+ @pytest.mark.parametrize("key", NEW_CONTROLS)
+ def test_explicit_null_is_no_control(monkeypatch, key):
+ client, fake = _budget_client(monkeypatch)
+ assert client.post("/v1/systemone", json={**REQ, key: None}).status_code == 200
+ assert not [name for name in NEW_CONTROLS if name in fake.calls[0]]
+
+
+ def test_min_confidence_zero_is_still_a_threshold(monkeypatch):
+ """`0.0` is falsy but is a value the caller chose; `if min_confidence:` would drop it."""
+ client, fake = _budget_client(monkeypatch)
+ assert client.post("/v1/systemone", json={**REQ, "min_confidence": 0.0}).status_code == 200
+ assert fake.calls[0]["min_confidence"] == 0.0
+
+
+ @pytest.mark.parametrize("key", ["lang", "lang_guess"])
+ @pytest.mark.parametrize("value", [True, 5, ["de"], {"code": "de"}])
+ def test_a_language_control_must_be_a_code_string(monkeypatch, key, value):
+ """`Router` stringifies a language hint, so `true` becomes the code `"true"`.
+
+ That is a real, non-English code, so the JSON boolean would decide the checkpoint. A callable
+ hint -- core's other accepted form -- cannot cross an HTTP body either, which leaves the code
+ string as the only form to accept here.
+ """
+ client, fake = _budget_client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, key: value})
+ assert r.status_code == 422, r.text
+ assert "must be a language code string" in r.json()["detail"]
+ assert not fake.calls, "a refused request must not run inference"
+
+
+ @pytest.mark.parametrize("value", [1.5, -0.1, True, "0.9"])
+ def test_min_confidence_is_validated_by_core(monkeypatch, value):
+ """The bounds belong to `laya.confidence`, so serve must report core's own message.
+
+ Comparing against the `ValueError` core raises for the same value is what keeps the accepted
+ range from drifting: a restated `[0.0, 1.0]` check here would pass CI on the day core widened
+ it and start rejecting requests the abstention gate would have honoured.
+ """
+ from laya.confidence import check_min_confidence
+
+ try:
+ check_min_confidence(value)
+ except ValueError as error:
+ expected = str(error)
+ else:
+ raise AssertionError("core accepted %r; this probe needs a rejecting value" % (value,))
+
+ client, fake = _budget_client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "min_confidence": value})
+ assert r.status_code == 422, r.text
+ assert r.json()["detail"] == expected
+ assert not fake.calls
+
+
+ def test_an_infinite_threshold_is_refused_by_core(monkeypatch):
+ """JSON writes no NaN, but a number literal can parse to infinity.
+
+ `check_min_confidence` tests `math.isfinite` for exactly this. A range check written here would
+ happen to reject it too, and then keep passing on the day core's rule changed -- so the request
+ is sent as raw bytes: Python cannot serialise an infinity back to JSON, and `1e999` is what a
+ client that never round-tripped through Python would actually put on the wire.
+ """
+ client, fake = _budget_client(monkeypatch)
+ raw = (json.dumps(REQ)[:-1] + ', "min_confidence": 1e999}').encode("utf-8")
+ r = client.post("/v1/systemone", content=raw,
+ headers={"Content-Type": "application/json"})
+ assert r.status_code == 422, r.text
+ assert "min_confidence must be a float in [0.0, 1.0]" in r.json()["detail"]
+ assert not fake.calls
+
+
+ def test_an_unknown_task_gets_cores_answer_not_a_500(monkeypatch):
+ """`task` is forwarded verbatim, as the CLI forwards `--task`, and core names the valid set.
+
+ `route()` normalises a task through `normalise_name`, whose `ValueError` the existing mapping
+ reports as a 422 carrying its message. Listing the accepted tasks in `serve` instead would be
+ a second registry to keep in step -- the failure mode #544 and #638 removed elsewhere.
+ """
+ from laya.router import normalise_name
+
+ class RoutingRouter(BudgetRouter):
+ def predict(self, state, questions, model=None, **kwargs):
+ if "task" in kwargs:
+ normalise_name(kwargs["task"]) # core's guard, called exactly as route() calls it
+ return super().predict(state, questions, model=model, **kwargs)
+
+ monkeypatch.delenv("LAYA_API_KEY", raising=False)
+ client = TestClient(create_app(router=RoutingRouter()), raise_server_exceptions=False)
+
+ r = client.post("/v1/systemone", json={**REQ, "task": "not-a-task"})
+ assert r.status_code == 422, r.text
+ assert "unknown model" in r.json()["detail"]
+
+ ok = client.post("/v1/systemone", json={**REQ, "task": "typed-decisions"})
+ assert ok.status_code == 200, ok.text
+
+
+ @pytest.mark.parametrize("key", BODY_REFUSALS)
+ def test_a_hook_control_is_refused_not_dropped(monkeypatch, key):
+ """All five used to be read into the body and ignored, so a client got a silent no.
+
+ `laya.integrations.langchain::_reject_remote_hooks` already refuses the same five on a node
+ with a `base_url`, on the grounds that a hook runs in the server's process. The endpoint now
+ says that itself instead of answering as though the control had been honoured.
+ """
+ value = {"hooks": [], "on_predict_start": "cache", "on_predict_end": "audit",
+ "hooks_raise": False, "hooks_timeout": 5.0}[key]
+ client, fake = _budget_client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, key: value})
+ assert r.status_code == 422, r.text
+ detail = r.json()["detail"]
+ assert key in detail
+ assert "cannot be sent to this endpoint" in detail
+ assert not fake.calls, "the refusal must come before inference"
+
+
+ def test_a_refusal_names_every_hook_control_sent(monkeypatch):
+ client, fake = _budget_client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "hooks_raise": True, "hooks_timeout": 5.0})
+ assert r.status_code == 422, r.text
+ detail = r.json()["detail"]
+ assert "hooks_raise" in detail and "hooks_timeout" in detail
+ assert not fake.calls
+
+
+ @pytest.mark.parametrize("key", BODY_REFUSALS)
+ def test_a_null_hook_control_is_not_a_refusal(monkeypatch, key):
+ """`{"hooks": null}` means "no per-call hooks", which is what core reads as inherit.
+
+ A Jev client that serialises its absent fields must keep working; only a value says the caller
+ asked for something this endpoint cannot do.
+ """
+ client, fake = _budget_client(monkeypatch)
+ assert client.post("/v1/systemone", json={**REQ, key: None}).status_code == 200
+
+
+ def test_http_api_page_documents_exactly_the_forwarded_controls():
+ """The request-body table and the code must not drift apart in either direction.
+
+ Read back out of the markdown rather than substring-matched: a row this parse cannot see is a
+ field the documentation stopped describing, and a documented field with no code behind it is
+ the same lie in the other direction. The five refusals have to be named where the page says
+ why they are refused, so the reason travels with the field list.
+ """
+ path = os.path.join(ROOT, "docs", "http-api.md")
+ with open(path, encoding="utf-8") as handle:
+ lines = handle.read().splitlines()
+
+ header = lines.index("| field | required | meaning |")
+ rows = []
+ for line in lines[header + 2:]:
+ if not line.strip():
+ break
+ rows.append(line.split("|")[1].strip().strip("`"))
+ documented = set(rows) - {"state", "questions"}
+ assert documented == set(BODY_CONTROLS), "table says %s, serve forwards %s" % (
+ sorted(documented), sorted(BODY_CONTROLS))
+
+ page = "\n".join(lines)
+ prose = page[page.index("`model`, `task`, `lang`, `lang_guess`"):]
+ for key in BODY_REFUSALS:
+ assert "`%s`" % key in prose, "%s is not named where the refusal is explained" % key