Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(examples): forward the per-call controls laya.serve forwards on /v1/systemone

Aashish248e2a9c2740
diff --git a/README.md b/README.md
index 328a6194..c1916d3b 100644
--- a/README.md
+++ b/README.md
@@ -288,6 +288,18 @@ A request body may carry `model` to pin a checkpoint instead of letting the rout
takes exactly the spellings `laya --model` takes: names, aliases and casing all resolve through
`laya.router`. `GET /models` lists the checkpoints and the aliases alongside them.
+Both `/predict` and `/predict/batch` also accept the four per-call controls `laya-serve`
+forwards on `/v1/systemone`: `lang_guess` (a soft ISO hint that participates in routing, where
+`lang` skips detection entirely), `max_len` and `head_max_len` (per-call token-budget overrides,
+capped by `LAYA_MAX_TOKEN_BUDGET`), and `min_confidence` (core's abstention threshold; the answer
+comes back with a `gate` field naming whether it fired). An absent key means "inherit what the
+Router was built with", so a caller never has to send them back as `null`; a body that names
+`hooks`, `hooks_raise`, `hooks_timeout`, `on_predict_start`, or `on_predict_end` is refused with
+422 rather than silently dropped, matching what `laya-serve` does for the same fields. On
+`/predict/batch`, `min_confidence` reaches `Router.predict_batch` as a call argument rather than
+a per-request key; the other three travel inside each request dict, so states in one batch may
+name different budgets.
+
`/predict/batch` accepts two optional body fields that control the shape of the forward passes
without changing any answer: `batch_size` (states per pass; omit it and the whole batch is one
pass) and `sort_by_length` (group similarly sized states so each pass pads to a shorter maximum —
diff --git a/examples/server.py b/examples/server.py
index 96c68a20..e070a6cd 100644
--- a/examples/server.py
+++ b/examples/server.py
@@ -69,6 +69,20 @@ MAX_STATE_CHARS = getattr(_laya_serve, "MAX_STATE_CHARS", 50_000)
MAX_CHOICE_OPTIONS = getattr(_laya_serve, "MAX_CHOICE_OPTIONS", 100)
MAX_SCORE_LEVELS = getattr(_laya_serve, "MAX_SCORE_LEVELS", 32)
MAX_TOTAL_OPTIONS = getattr(_laya_serve, "MAX_TOTAL_OPTIONS", 512)
+DEFAULT_MAX_TOKEN_BUDGET = getattr(_laya_serve, "DEFAULT_MAX_TOKEN_BUDGET", 8192)
+
+# The set of controls a client may put on the body, and the set that must be refused rather than
+# silently dropped -- both read from `laya.serve` so the demo cannot drift from the server it
+# demonstrates. `BODY_CONTROLS` is what the shipped `/v1/systemone` forwards; `BODY_REFUSALS` is
+# the hook surface (`hooks`, `hooks_raise`, `hooks_timeout`, `on_predict_start`, `on_predict_end`)
+# that a `ServerApp` cannot receive over HTTP but the Router does take as call arguments -- the
+# demo has no hook receiver, so those are refused the same way.
+BODY_CONTROLS = tuple(getattr(_laya_serve, "BODY_CONTROLS",
+ ("model", "max_len", "head_max_len", "task", "lang",
+ "lang_guess", "min_confidence")))
+BODY_REFUSALS = tuple(getattr(_laya_serve, "BODY_REFUSALS",
+ ("hooks", "on_predict_start", "on_predict_end",
+ "hooks_raise", "hooks_timeout")))
def _fallback_state_length(state: Any) -> int:
@@ -149,6 +163,87 @@ def _check_model(v: Optional[str]) -> Optional[str]:
raise ValueError("%s (or omit it)" % error) from None
+# The four per-call controls the shipped `/v1/systemone` validates and forwards. Each of the
+# validators below reads the corresponding one out of `laya.serve` by `getattr`, falling back to
+# a body-compatible local so this demo still refuses a bad control on a laya that predates the
+# shared helpers. The fallbacks mirror `laya.serve` exactly, so the accepted range and the 422
+# detail a caller sees on `main` are the same as the shipped server's.
+_resolve_max_token_budget = getattr(_laya_serve, "_resolve_max_token_budget",
+ lambda: DEFAULT_MAX_TOKEN_BUDGET)
+
+
+def _fallback_refuse_body_refusals(body: Dict[str, Any]) -> None:
+ given = sorted(key for key in BODY_REFUSALS if key in body and body[key] is not None)
+ if given:
+ raise HTTPException(
+ status_code=422,
+ detail="%s run inside the server process and cannot be sent to this endpoint; "
+ "install them where laya-serve runs, or drop them" % ", ".join(given))
+
+
+def _fallback_validate_budget_param(body: Dict[str, Any], key: str, max_cap: int):
+ if key not in body:
+ return None
+ val = body[key]
+ if val is None:
+ return None
+ if not isinstance(val, int) or isinstance(val, bool):
+ raise HTTPException(status_code=422, detail="%s must be an integer" % key)
+ if val <= 0:
+ raise HTTPException(status_code=422, detail="%s must be a positive integer" % key)
+ if val > max_cap:
+ raise HTTPException(status_code=422,
+ detail="%s exceeds server limit (%d > %d)" % (key, val, max_cap))
+ return val
+
+
+def _fallback_validate_language_param(body: Dict[str, Any], key: str):
+ val = body.get(key)
+ if val is None:
+ return None
+ if not isinstance(val, str):
+ raise HTTPException(status_code=422,
+ detail='%s must be a language code string such as "de", or null' % key)
+ return val
+
+
+def _fallback_validate_min_confidence(body: Dict[str, Any]):
+ if body.get("min_confidence") is None:
+ return None
+ from laya.confidence import check_min_confidence
+ try:
+ return check_min_confidence(body["min_confidence"])
+ except ValueError as error:
+ raise HTTPException(status_code=422, detail=str(error)) from None
+
+
+_refuse_body_refusals = getattr(_laya_serve, "_refuse_body_refusals",
+ _fallback_refuse_body_refusals)
+_validate_budget_param = getattr(_laya_serve, "_validate_budget_param",
+ _fallback_validate_budget_param)
+_validate_language_param = getattr(_laya_serve, "_validate_language_param",
+ _fallback_validate_language_param)
+_validate_min_confidence = getattr(_laya_serve, "_validate_min_confidence",
+ _fallback_validate_min_confidence)
+
+
+def _per_call_controls(req: Union["PredictRequest", "BatchRequest"]) -> Dict[str, Any]:
+ """The set-control dict to spread into a request -- absent controls stay absent.
+
+ `Router.predict` and `route_batch` read an absent argument as "inherit what the Router was
+ built with", so sending `None` here would override a deployment's `Router(lang_guess=...)`
+ or `Router(min_confidence=...)` with the demo's default. Only controls the client actually
+ sent -- including `min_confidence=0.0`, which is falsy but a real ask -- travel on.
+ """
+ out: Dict[str, Any] = {}
+ for key in ("model", "task", "lang", "lang_guess", "max_len", "head_max_len",
+ "min_confidence"):
+ value = getattr(req, key, None)
+ if value is not None:
+ out[key] = value
+ return out
+
+
class Question(BaseModel):
"""One question in the `questions` mapping."""
@@ -193,9 +288,45 @@ class PredictRequest(BaseModel):
)
task: Optional[str] = None
lang: Optional[str] = Field(default=None, description="ISO code hint; skips detection")
+ # The four controls laya.serve's `/v1/systemone` forwards on the same body and this demo did
+ # not accept. `max_len` / `head_max_len` override the token budget for one call;
+ # `lang_guess` is the soft hint that routes when `lang` is absent; `min_confidence` gates
+ # the answer to an abstention at core's threshold. All four are conditional at the call site:
+ # an absent key means "inherit what the Router was built with", which sending null would
+ # override with the server's default.
+ lang_guess: Optional[str] = Field(
+ default=None,
+ description="Soft ISO language hint that participates in routing; `lang` skips detection")
+ max_len: Optional[int] = Field(
+ default=None,
+ description="Per-call state token budget; capped by LAYA_MAX_TOKEN_BUDGET (%d by default)"
+ % DEFAULT_MAX_TOKEN_BUDGET)
+ head_max_len: Optional[int] = Field(
+ default=None,
+ description="Per-call question-head token budget; capped by LAYA_MAX_TOKEN_BUDGET (%d "
+ "by default)" % DEFAULT_MAX_TOKEN_BUDGET)
+ min_confidence: Optional[float] = Field(
+ default=None,
+ description="Abstain when answer_confidence is below this threshold (0.0-1.0)")
_v_model = field_validator("model")(classmethod(lambda cls, v: _check_model(v)))
+ @model_validator(mode="before")
+ @classmethod
+ def _check_controls(cls, data: Any) -> Any:
+ # The hook surface, the budget caps, the language-code shape and the abstention range
+ # are checked by `laya.serve`'s own validators, called here on the raw body, so this
+ # demo refuses the same bodies with the same 422 detail as the server it demonstrates.
+ if isinstance(data, dict):
+ _refuse_body_refusals(data)
+ cap = _resolve_max_token_budget()
+ _validate_budget_param(data, "max_len", cap)
+ _validate_budget_param(data, "head_max_len", cap)
+ _validate_language_param(data, "lang")
+ _validate_language_param(data, "lang_guess")
+ _validate_min_confidence(data)
+ return data
+
@field_validator("state")
@classmethod
def _non_empty(cls, v):
@@ -211,6 +342,11 @@ class BatchRequest(BaseModel):
+ _model_names())
task: Optional[str] = None
lang: Optional[str] = None
+ lang_guess: Optional[str] = Field(default=None, description="See PredictRequest.lang_guess")
+ max_len: Optional[int] = Field(default=None, description="See PredictRequest.max_len")
+ head_max_len: Optional[int] = Field(default=None, description="See PredictRequest.head_max_len")
+ min_confidence: Optional[float] = Field(default=None,
+ description="See PredictRequest.min_confidence")
# The batch shape, not a per-state control: these size and group the forward passes rather
# than saying what a state means. `sort_by_length` only reorders a batch that is split into
# more than one pass, so it needs a `batch_size` strictly between 1 and the state count.
@@ -224,6 +360,22 @@ class BatchRequest(BaseModel):
_v_model = field_validator("model")(classmethod(lambda cls, v: _check_model(v)))
+ @model_validator(mode="before")
+ @classmethod
+ def _check_controls(cls, data: Any) -> Any:
+ # Same four refusals and same four controls as PredictRequest -- a body that names
+ # `hooks` on a batch or a `max_len` past the cap is refused here the way the shipped
+ # server refuses it, not silently dropped on the way into the per-request dict.
+ if isinstance(data, dict):
+ _refuse_body_refusals(data)
+ cap = _resolve_max_token_budget()
+ _validate_budget_param(data, "max_len", cap)
+ _validate_budget_param(data, "head_max_len", cap)
+ _validate_language_param(data, "lang")
+ _validate_language_param(data, "lang_guess")
+ _validate_min_confidence(data)
+ return data
+
# --------------------------------------------------------------------------- #
# App
@@ -409,9 +561,7 @@ def predict(req: PredictRequest) -> Dict[str, Any]:
return _predict(
req.state,
_questions(req.questions),
- model=req.model,
- task=req.task,
- lang=req.lang,
+ **_per_call_controls(req),
)
except HTTPException:
raise
@@ -431,8 +581,11 @@ def predict_batch(req: BatchRequest) -> Dict[str, Any]:
questions = _questions(req.questions)
# Only the controls that were actually set: `route_batch` reads them with `.get`, so an
# omitted key and an explicit null mean the same thing, and the request dicts stay minimal.
- controls = {key: value for key, value in (("model", req.model), ("task", req.task),
- ("lang", req.lang)) if value is not None}
+ # `min_confidence` is the exception: `Router.predict_batch` takes it as a call argument, not
+ # a per-request key, so `route_batch` would drop it from the request dict. On the batch path
+ # it moves below into `shape`; on the per-state fallback below it rejoins the controls, since
+ # `Router.predict` does read it as a call argument.
+ controls = {k: v for k, v in _per_call_controls(req).items() if k != "min_confidence"}
requests = [{"state": state, "questions": questions, **controls} for state in req.states]
# The batch shape is an argument of the batch call, not a control that travels per request: the
# fallback below answers one state at a time, where there is no group to size or reorder. Sent
@@ -442,6 +595,8 @@ def predict_batch(req: BatchRequest) -> Dict[str, Any]:
shape["batch_size"] = req.batch_size
if req.sort_by_length:
shape["sort_by_length"] = True
+ if req.min_confidence is not None:
+ shape["min_confidence"] = req.min_confidence
try:
# One call, not one per state: `Router.predict_batch` routes the whole batch, groups it by
# checkpoint and shares a forward pass across states that carry the same question schema --
@@ -452,9 +607,14 @@ def predict_batch(req: BatchRequest) -> Dict[str, Any]:
# its answer. Fall back to the per-state path to keep the documented envelope: N results,
# with `{"index": i, "error": ...}` only where a state genuinely failed.
results = []
+ # On the fallback `Router.predict` takes min_confidence as a call argument, so it rejoins
+ # the controls the batch path pulled out above.
+ per_state = dict(controls)
+ if req.min_confidence is not None:
+ per_state["min_confidence"] = req.min_confidence
for i, state in enumerate(req.states):
try:
- results.append(_predict(state, questions, **controls))
+ results.append(_predict(state, questions, **per_state))
except HTTPException as item_exc: # a caller-facing status (413, 422, 503) is safe
results.append({"index": i, "error": "HTTPException: %d: %s"
% (item_exc.status_code, item_exc.detail)})
diff --git a/tests/test_example_server_per_call_controls.py b/tests/test_example_server_per_call_controls.py
new file mode 100644
index 00000000..b9d4da5a
--- /dev/null
+++ b/tests/test_example_server_per_call_controls.py
@@ -0,0 +1,359 @@
+"""Regression: examples/server.py must forward the same per-call controls laya.serve does.
+
+`laya/serve.py::BODY_CONTROLS` is the set of caller-side knobs the shipped `/v1/systemone`
+validates and forwards to the Router -- `model`, `max_len`, `head_max_len`, `task`, `lang`,
+`lang_guess`, `min_confidence` -- and it refuses `BODY_REFUSALS` (the hook surface that cannot
+cross an HTTP body) rather than silently dropping it. `laya/cli.py`, `laya/mcp/tools.py` and the
+`laya.integrations.*` wrappers each forward that same set.
+
+The demo server did not. Its `PredictRequest` took only `model`/`task`/`lang`, and `BatchRequest`
+added the batch shape but nothing else, so a caller who asked for a per-call token budget, a soft
+language hint, or an abstention threshold got the *same* answer as one who asked for nothing. On a
+deployment that runs `Router(lang_guess=...)` or `Router(min_confidence=...)`, the demo could not
+override it per call; a client who asked for `max_len` past `LAYA_MAX_TOKEN_BUDGET` was served
+rather than refused; and a body that named `hooks` -- the surface this file has no way to receive
+-- was silently ignored instead of answered with the same 422 the shipped server gives.
+
+Scope: what the *handlers* forward on `Router.predict` and `Router.predict_batch`, and what the
+*validators* refuse before the Router is touched. The two are pinned together: a body that names
+`max_len` and is refused (past the cap) never reaches the recording Router, while one that names
+`max_len=256` does, and the call the handler made carries the value -- not `None`, since core
+reads an absent argument as "inherit what the deployment built with" and a null would override a
+Router's own `lang_guess`/`min_confidence` with the demo's default.
+
+`min_confidence` is the one asymmetry, and it has a reason: `Router.predict_batch` takes it as a
+call argument, not a per-request key (`laya/router.py:961`), so on `/predict/batch` it rides on the
+call shape rather than the request dicts. The per-state fallback in that endpoint goes through
+`Router.predict`, which does take it as an argument, so on the retry the same value rejoins the
+controls -- the fallback is one call per state and each carries the ask.
+
+Driven over HTTP through TestClient, on a recording stand-in at `demo.ROUTER`. No weights load.
+
+Run: python tests/test_example_server_per_call_controls.py
+"""
+import os
+import sys
+
+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
+os.environ.setdefault("LAYA_PRELOAD", "0")
+os.environ.pop("LAYA_MAX_TOKEN_BUDGET", None)
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+sys.path.insert(0, os.path.join(ROOT, "examples"))
+
+PASS, FAIL = [], []
+
+
+def ok(name, cond, detail=""):
+ (PASS if cond else FAIL).append(
+ "%s%s" % (name, (" -- " + detail) if detail and not cond else ""))
+ print(" %s %s%s" % ("PASS" if cond else "FAIL", name,
+ (" " + detail) if detail and not cond else ""), flush=True)
+
+
+def main():
+ try:
+ from fastapi.testclient import TestClient
+ except ImportError as exc:
+ if (getattr(exc, "name", None) or "").split(".")[0] not in ("fastapi", "httpx", "starlette"):
+ raise
+ print("SKIP: fastapi/httpx not installed -- pip install laya[serve] httpx")
+ return 0
+ try:
+ import server as demo
+ except ImportError as exc:
+ if (getattr(exc, "name", None) or "").split(".")[0] not in (
+ "fastapi", "httpx", "starlette", "multipart"):
+ raise
+ print("SKIP: examples/server.py needs the serve extra -- pip install laya[serve]")
+ return 0
+
+ from laya.serve import BODY_CONTROLS, BODY_REFUSALS
+
+ class RecordingRouter:
+ """Answers like the Router does, and remembers how it was asked."""
+
+ def __init__(self):
+ self.predict_calls = []
+ self.batch_calls = []
+
+ def predict(self, state, questions, **kw):
+ self.predict_calls.append((state, dict(kw)))
+ return {"answers": {"a": {"type": "noul", "choice": False}},
+ "model": "english", "state": state, "questions": questions}
+
+ def predict_batch(self, requests, **kw):
+ self.batch_calls.append((list(requests), dict(kw)))
+ return [{"answers": {"a": {"type": "noul", "choice": False}},
+ "model": "english", "state": r["state"], "questions": r["questions"]}
+ for r in requests]
+
+ def new_recorder():
+ r = RecordingRouter()
+ demo.ROUTER = r
+ return r
+
+ client = TestClient(demo.app, raise_server_exceptions=False)
+ one = {"a": {"type": "noul", "instructions": "x"}}
+
+ # --- what the two surfaces forward is the same set, drawn from laya.serve ----------
+ # PredictRequest's public fields must cover BODY_CONTROLS exactly: this file's own claim
+ # to be "the demo of the shipped server" fails if it stops accepting a control core takes.
+ per_call_set = set(BODY_CONTROLS)
+ predict_fields = set(demo.PredictRequest.model_fields) - {"state", "questions"}
+ batch_fields = set(demo.BatchRequest.model_fields) - {"states", "questions",
+ "batch_size", "sort_by_length"}
+ ok("PredictRequest accepts every key in laya.serve.BODY_CONTROLS",
+ per_call_set <= predict_fields,
+ "BODY_CONTROLS=%r, PredictRequest fields=%r" % (sorted(per_call_set),
+ sorted(predict_fields)))
+ ok("BatchRequest accepts the same set, not a subset that drifts from the single path",
+ per_call_set <= batch_fields,
+ "missing from BatchRequest: %r" % sorted(per_call_set - batch_fields))
+ ok("and the two models expose the same per-call surface, so a client sees no drift",
+ predict_fields == batch_fields,
+ "predict=%r vs batch=%r" % (sorted(predict_fields), sorted(batch_fields)))
+
+ # --- /predict forwards each control when the caller set it -----------------------
+ r = new_recorder()
+ client.post("/predict", json={"state": "hello", "questions": one,
+ "max_len": 256})
+ ok("max_len reaches Router.predict when set",
+ r.predict_calls and r.predict_calls[0][1].get("max_len") == 256,
+ str(r.predict_calls[0][1] if r.predict_calls else "no call"))
+
+ r = new_recorder()
+ client.post("/predict", json={"state": "hello", "questions": one,
+ "head_max_len": 64})
+ ok("head_max_len reaches Router.predict when set",
+ r.predict_calls and r.predict_calls[0][1].get("head_max_len") == 64)
+
+ r = new_recorder()
+ client.post("/predict", json={"state": "hello", "questions": one,
+ "lang_guess": "de"})
+ ok("lang_guess reaches Router.predict when set",
+ r.predict_calls and r.predict_calls[0][1].get("lang_guess") == "de")
+
+ r = new_recorder()
+ client.post("/predict", json={"state": "hello", "questions": one,
+ "min_confidence": 0.85})
+ ok("min_confidence reaches Router.predict when set",
+ r.predict_calls and r.predict_calls[0][1].get("min_confidence") == 0.85)
+
+ r = new_recorder()
+ client.post("/predict", json={"state": "hello", "questions": one,
+ "model": "english", "task": "routing", "lang": "fr",
+ "lang_guess": "de", "max_len": 512, "head_max_len": 32,
+ "min_confidence": 0.5})
+ ok("and every control on a single body arrives together, none overwriting another",
+ r.predict_calls
+ and r.predict_calls[0][1] == {"model": "english", "task": "routing", "lang": "fr",
+ "lang_guess": "de", "max_len": 512, "head_max_len": 32,
+ "min_confidence": 0.5},
+ str(r.predict_calls[0][1] if r.predict_calls else "no call"))
+
+ # --- unset controls are absent, never sent as None ------------------------------
+ # `Router.predict` reads an absent argument as "inherit what the deployment built with",
+ # so passing None here would override a `Router(lang_guess=...)` or a
+ # `Router(min_confidence=...)` with the demo's own default.
+ r = new_recorder()
+ client.post("/predict", json={"state": "hello", "questions": one})
+ kw = r.predict_calls[0][1] if r.predict_calls else None
+ ok("an unset control is absent from the call, not sent as None",
+ kw is not None and not any(kw.get(k, "__sentinel__") is None
+ for k in ("max_len", "head_max_len", "lang_guess",
+ "min_confidence", "task", "lang", "model")),
+ str(kw))
+
+ # `min_confidence=0.0` is falsy but a real ask -- it means "gate at zero", which is not
+ # the same as "no gate". Any `if value:` guard would drop it; only `if value is not None`
+ # is correct here.
+ r = new_recorder()
+ client.post("/predict", json={"state": "hello", "questions": one,
+ "min_confidence": 0.0})
+ ok("min_confidence=0.0 is still forwarded -- falsy is not absent",
+ r.predict_calls and "min_confidence" in r.predict_calls[0][1]
+ and r.predict_calls[0][1]["min_confidence"] == 0.0,
+ str(r.predict_calls[0][1] if r.predict_calls else "no call"))
+
+ # --- the shipped server's refusals are also refused here -----------------------
+ # A body that names `hooks` cannot cross this surface: there is no receiver on the
+ # demo, so a caller who asks for one today gets an answer that ignores their ask. The
+ # shipped server 422s; the demo must too, on both endpoints.
+ for key in BODY_REFUSALS:
+ r = new_recorder()
+ resp = client.post("/predict", json={"state": "hello", "questions": one, key: []})
+ ok("a body that names %r is 422, not silently ignored" % key,
+ resp.status_code == 422 and not r.predict_calls,
+ "%s, router calls=%d" % (resp.status_code, len(r.predict_calls)))
+
+ r = new_recorder()
+ resp = client.post("/predict/batch", json={"states": ["a", "b"], "questions": one,
+ "hooks_timeout": 0.5})
+ ok("the refusal covers /predict/batch too",
+ resp.status_code == 422 and not r.batch_calls,
+ "%s, batch calls=%d" % (resp.status_code, len(r.batch_calls)))
+
+ # --- max_len is capped by LAYA_MAX_TOKEN_BUDGET, refused before inference ------
+ cap = demo._resolve_max_token_budget()
+ r = new_recorder()
+ resp = client.post("/predict", json={"state": "hello", "questions": one,
+ "max_len": cap + 1})
+ ok("a max_len past the server cap is a 422, and the Router is never asked",
+ resp.status_code == 422 and not r.predict_calls,
+ "%s, calls=%d" % (resp.status_code, len(r.predict_calls)))
+
+ r = new_recorder()
+ resp = client.post("/predict", json={"state": "hello", "questions": one,
+ "head_max_len": cap + 1})
+ ok("the same cap covers head_max_len",
+ resp.status_code == 422 and not r.predict_calls)
+
+ # The cap is read at call time, so LAYA_MAX_TOKEN_BUDGET actually moves it: the demo
+ # does not carry a copy of the number.
+ os.environ["LAYA_MAX_TOKEN_BUDGET"] = "2048"
+ try:
+ r = new_recorder()
+ resp = client.post("/predict", json={"state": "hello", "questions": one,
+ "max_len": 4096})
+ ok("LAYA_MAX_TOKEN_BUDGET is what caps the request, not a number typed in this file",
+ resp.status_code == 422 and not r.predict_calls,
+ "%s" % resp.status_code)
+ r = new_recorder()
+ ok("and the same value under the raised cap is accepted and forwarded",
+ client.post("/predict", json={"state": "hello", "questions": one,
+ "max_len": 1024}).status_code == 200
+ and r.predict_calls[0][1]["max_len"] == 1024)
+ finally:
+ os.environ.pop("LAYA_MAX_TOKEN_BUDGET", None)
+
+ # --- type refusals, mirroring laya.serve's validators -------------------------
+ # A JSON `true` where the Router expects a language code would become `"true"` through
+ # core's `_english_from_code` -- a real, non-English code that decides the checkpoint.
+ # `Optional[str]` in the field type also refuses a bool, so a plain status check cannot
+ # tell the two paths apart; the assertion below requires laya.serve's own wording in
+ # the 422 body, which only fires when this file actually calls `_validate_language_param`.
+ for key in ("lang", "lang_guess"):
+ r = new_recorder()
+ resp = client.post("/predict", json={"state": "hello", "questions": one, key: True})
+ detail = str(resp.json().get("detail", ""))
+ ok("%s: a JSON bool is refused before the Router is asked" % key,
+ resp.status_code == 422 and not r.predict_calls,
+ "%s, calls=%d" % (resp.status_code, len(r.predict_calls)))
+ ok("%s: the 422 detail is laya.serve's own wording, not pydantic's field error" % key,
+ "language code string" in detail, detail[:180])
+
+ r = new_recorder()
+ resp = client.post("/predict", json={"state": "hello", "questions": one,
+ "min_confidence": True})
+ ok("min_confidence: a JSON bool is refused -- it is not a threshold",
+ resp.status_code == 422 and not r.predict_calls,
+ "%s, calls=%d" % (resp.status_code, len(r.predict_calls)))
+
+ r = new_recorder()
+ resp = client.post("/predict", json={"state": "hello", "questions": one,
+ "min_confidence": 1.5})
+ ok("min_confidence outside core's [0, 1] is refused before an inference slot",
+ resp.status_code == 422 and not r.predict_calls)
+
+ r = new_recorder()
+ resp = client.post("/predict", json={"state": "hello", "questions": one,
+ "max_len": "abc"})
+ ok("a non-integer max_len is refused, not coerced",
+ resp.status_code == 422 and not r.predict_calls)
+
+ r = new_recorder()
+ resp = client.post("/predict", json={"state": "hello", "questions": one,
+ "max_len": -1})
+ ok("a non-positive max_len is refused, matching laya.serve._validate_budget_param",
+ resp.status_code == 422 and not r.predict_calls)
+
+ # --- /predict/batch forwards the same set -------------------------------------
+ # On the batch path the per-request controls ride in each request dict (so `route_batch`
+ # and `predict_batch` read them per state), while `min_confidence` is a call-level
+ # argument to `Router.predict_batch` -- see laya/router.py:961.
+ r = new_recorder()
+ states = ["s1", "s2", "s3"]
+ client.post("/predict/batch", json={"states": states, "questions": one,
+ "lang": "fr", "lang_guess": "de",
+ "max_len": 512, "head_max_len": 64,
+ "min_confidence": 0.75,
+ "batch_size": 2, "sort_by_length": True})
+ reqs, shape = r.batch_calls[0] if r.batch_calls else ([], {})
+ ok("each request on the batch carries the caller's max_len/head_max_len/lang/lang_guess",
+ all(rr.get("max_len") == 512 and rr.get("head_max_len") == 64
+ and rr.get("lang") == "fr" and rr.get("lang_guess") == "de" for rr in reqs),
+ str(reqs[:1]))
+ ok("min_confidence rides on the batch call as a shape argument, not per request",
+ shape.get("min_confidence") == 0.75
+ and not any("min_confidence" in rr for rr in reqs),
+ "shape=%r" % shape)
+ ok("batch_size and sort_by_length still reach the Router alongside it",
+ shape.get("batch_size") == 2 and shape.get("sort_by_length") is True)
+
+ # Unset controls on the batch path stay absent too, for the same "inherit" reason.
+ r = new_recorder()
+ client.post("/predict/batch", json={"states": states, "questions": one})
+ reqs, shape = r.batch_calls[0] if r.batch_calls else ([], {})
+ ok("a plain batch sends no null controls and no shape",
+ reqs and not any(k in rr for rr in reqs for k in
+ ("max_len", "head_max_len", "lang", "lang_guess", "min_confidence",
+ "task", "model"))
+ and not shape,
+ "reqs=%r shape=%r" % (reqs[:1], shape))
+
+ # --- the fallback keeps the ask on every per-state retry ----------------------
+ # When Router.predict_batch fails, the endpoint retries each state through Router.predict,
+ # which *does* take min_confidence as a call argument. If the retry dropped it, one bad
+ # state would silently cost the neighbours their abstention gate.
+ class HalfRouter(RecordingRouter):
+ def predict_batch(self, requests, **kw):
+ self.batch_calls.append((list(requests), dict(kw)))
+ raise ValueError("batch forward failed")
+
+ r = HalfRouter()
+ demo.ROUTER = r
+ body = client.post("/predict/batch", json={"states": states, "questions": one,
+ "max_len": 256, "min_confidence": 0.75}).json()
+ ok("the fallback retried every state",
+ len(body.get("results", [])) == 3 and len(r.predict_calls) == 3)
+ ok("and each retry carries the caller's max_len",
+ all(c[1].get("max_len") == 256 for c in r.predict_calls),
+ str([c[1] for c in r.predict_calls][:1]))
+ ok("and each retry carries the abstention threshold, this time as a call argument",
+ all(c[1].get("min_confidence") == 0.75 for c in r.predict_calls),
+ str([c[1] for c in r.predict_calls][:1]))
+ # batch_size and sort_by_length belong to the forward-pass shape only; the fallback is one
+ # state per call, so leaking them into `Router.predict(...)` would TypeError every retry.
+ ok("and the fallback does not leak batch shape into Router.predict",
+ not any("batch_size" in c[1] or "sort_by_length" in c[1] for c in r.predict_calls),
+ str([c[1] for c in r.predict_calls][:1]))
+
+ # --- parity with laya.serve's refusal wording ---------------------------------
+ # The detail must be the shipped server's, not a paraphrase: a client that reads the
+ # 422 body gets the same "install them where laya-serve runs" guidance from both.
+ resp = client.post("/predict", json={"state": "hello", "questions": one, "hooks": []})
+ ok("the 422 detail is laya.serve's own wording, not a copy",
+ resp.status_code == 422
+ and "run inside the server process" in str(resp.json().get("detail", "")),
+ str(resp.json())[:220])
+
+ # --- the two endpoints refuse identically -------------------------------------
+ # A client who POSTs `{"hooks_timeout": 0.5}` to the batch endpoint must not get a
+ # different answer than one who POSTs the same field to the single endpoint.
+ for key in BODY_REFUSALS:
+ single = client.post("/predict", json={"state": "hi", "questions": one, key: 1})
+ batch = client.post("/predict/batch", json={"states": ["a"], "questions": one, key: 1})
+ ok("%s refused the same way on both endpoints" % key,
+ single.status_code == batch.status_code == 422,
+ "single=%s batch=%s" % (single.status_code, batch.status_code))
+
+ demo.ROUTER = None
+ print()
+ print("%d passed, %d failed" % (len(PASS), len(FAIL)))
+ return 0 if not FAIL else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())