Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(examples): forward the per-call controls laya.serve forwards on /v1/systemone
Aashish248e2a9c2740
diff --git a/README.md b/README.mdindex 328a6194..c1916d3b 100644--- a/README.md+++ b/README.md@@ -288,6 +288,18 @@ A request body may carry `model` to pin a checkpoint instead of letting the routtakes exactly the spellings `laya --model` takes: names, aliases and casing all resolve through`laya.router`. `GET /models` lists the checkpoints and the aliases alongside them.+Both `/predict` and `/predict/batch` also accept the four per-call controls `laya-serve`+forwards on `/v1/systemone`: `lang_guess` (a soft ISO hint that participates in routing, where+`lang` skips detection entirely), `max_len` and `head_max_len` (per-call token-budget overrides,+capped by `LAYA_MAX_TOKEN_BUDGET`), and `min_confidence` (core's abstention threshold; the answer+comes back with a `gate` field naming whether it fired). An absent key means "inherit what the+Router was built with", so a caller never has to send them back as `null`; a body that names+`hooks`, `hooks_raise`, `hooks_timeout`, `on_predict_start`, or `on_predict_end` is refused with+422 rather than silently dropped, matching what `laya-serve` does for the same fields. On+`/predict/batch`, `min_confidence` reaches `Router.predict_batch` as a call argument rather than+a per-request key; the other three travel inside each request dict, so states in one batch may+name different budgets.+`/predict/batch` accepts two optional body fields that control the shape of the forward passeswithout changing any answer: `batch_size` (states per pass; omit it and the whole batch is onepass) and `sort_by_length` (group similarly sized states so each pass pads to a shorter maximum —diff --git a/examples/server.py b/examples/server.pyindex 96c68a20..e070a6cd 100644--- a/examples/server.py+++ b/examples/server.py@@ -69,6 +69,20 @@ MAX_STATE_CHARS = getattr(_laya_serve, "MAX_STATE_CHARS", 50_000)MAX_CHOICE_OPTIONS = getattr(_laya_serve, "MAX_CHOICE_OPTIONS", 100)MAX_SCORE_LEVELS = getattr(_laya_serve, "MAX_SCORE_LEVELS", 32)MAX_TOTAL_OPTIONS = getattr(_laya_serve, "MAX_TOTAL_OPTIONS", 512)+DEFAULT_MAX_TOKEN_BUDGET = getattr(_laya_serve, "DEFAULT_MAX_TOKEN_BUDGET", 8192)++# The set of controls a client may put on the body, and the set that must be refused rather than+# silently dropped -- both read from `laya.serve` so the demo cannot drift from the server it+# demonstrates. `BODY_CONTROLS` is what the shipped `/v1/systemone` forwards; `BODY_REFUSALS` is+# the hook surface (`hooks`, `hooks_raise`, `hooks_timeout`, `on_predict_start`, `on_predict_end`)+# that a `ServerApp` cannot receive over HTTP but the Router does take as call arguments -- the+# demo has no hook receiver, so those are refused the same way.+BODY_CONTROLS = tuple(getattr(_laya_serve, "BODY_CONTROLS",+ ("model", "max_len", "head_max_len", "task", "lang",+ "lang_guess", "min_confidence")))+BODY_REFUSALS = tuple(getattr(_laya_serve, "BODY_REFUSALS",+ ("hooks", "on_predict_start", "on_predict_end",+ "hooks_raise", "hooks_timeout")))def _fallback_state_length(state: Any) -> int:@@ -149,6 +163,87 @@ def _check_model(v: Optional[str]) -> Optional[str]:raise ValueError("%s (or omit it)" % error) from None+# The four per-call controls the shipped `/v1/systemone` validates and forwards. Each of the+# validators below reads the corresponding one out of `laya.serve` by `getattr`, falling back to+# a body-compatible local so this demo still refuses a bad control on a laya that predates the+# shared helpers. The fallbacks mirror `laya.serve` exactly, so the accepted range and the 422+# detail a caller sees on `main` are the same as the shipped server's.+_resolve_max_token_budget = getattr(_laya_serve, "_resolve_max_token_budget",+ lambda: DEFAULT_MAX_TOKEN_BUDGET)+++def _fallback_refuse_body_refusals(body: Dict[str, Any]) -> None:+ given = sorted(key for key in BODY_REFUSALS if key in body and body[key] is not None)+ if given:+ raise HTTPException(+ status_code=422,+ detail="%s run inside the server process and cannot be sent to this endpoint; "+ "install them where laya-serve runs, or drop them" % ", ".join(given))+++def _fallback_validate_budget_param(body: Dict[str, Any], key: str, max_cap: int):+ if key not in body:+ return None+ val = body[key]+ if val is None:+ return None+ if not isinstance(val, int) or isinstance(val, bool):+ raise HTTPException(status_code=422, detail="%s must be an integer" % key)+ if val <= 0:+ raise HTTPException(status_code=422, detail="%s must be a positive integer" % key)+ if val > max_cap:+ raise HTTPException(status_code=422,+ detail="%s exceeds server limit (%d > %d)" % (key, val, max_cap))+ return val+++def _fallback_validate_language_param(body: Dict[str, Any], key: str):+ val = body.get(key)+ if val is None:+ return None+ if not isinstance(val, str):+ raise HTTPException(status_code=422,+ detail='%s must be a language code string such as "de", or null' % key)+ return val+++def _fallback_validate_min_confidence(body: Dict[str, Any]):+ if body.get("min_confidence") is None:+ return None+ from laya.confidence import check_min_confidence+ try:+ return check_min_confidence(body["min_confidence"])+ except ValueError as error:+ raise HTTPException(status_code=422, detail=str(error)) from None+++_refuse_body_refusals = getattr(_laya_serve, "_refuse_body_refusals",+ _fallback_refuse_body_refusals)+_validate_budget_param = getattr(_laya_serve, "_validate_budget_param",+ _fallback_validate_budget_param)+_validate_language_param = getattr(_laya_serve, "_validate_language_param",+ _fallback_validate_language_param)+_validate_min_confidence = getattr(_laya_serve, "_validate_min_confidence",+ _fallback_validate_min_confidence)+++def _per_call_controls(req: Union["PredictRequest", "BatchRequest"]) -> Dict[str, Any]:+ """The set-control dict to spread into a request -- absent controls stay absent.++ `Router.predict` and `route_batch` read an absent argument as "inherit what the Router was+ built with", so sending `None` here would override a deployment's `Router(lang_guess=...)`+ or `Router(min_confidence=...)` with the demo's default. Only controls the client actually+ sent -- including `min_confidence=0.0`, which is falsy but a real ask -- travel on.+ """+ out: Dict[str, Any] = {}+ for key in ("model", "task", "lang", "lang_guess", "max_len", "head_max_len",+ "min_confidence"):+ value = getattr(req, key, None)+ if value is not None:+ out[key] = value+ return out++class Question(BaseModel):"""One question in the `questions` mapping."""@@ -193,9 +288,45 @@ class PredictRequest(BaseModel):)task: Optional[str] = Nonelang: Optional[str] = Field(default=None, description="ISO code hint; skips detection")+ # The four controls laya.serve's `/v1/systemone` forwards on the same body and this demo did+ # not accept. `max_len` / `head_max_len` override the token budget for one call;+ # `lang_guess` is the soft hint that routes when `lang` is absent; `min_confidence` gates+ # the answer to an abstention at core's threshold. All four are conditional at the call site:+ # an absent key means "inherit what the Router was built with", which sending null would+ # override with the server's default.+ lang_guess: Optional[str] = Field(+ default=None,+ description="Soft ISO language hint that participates in routing; `lang` skips detection")+ max_len: Optional[int] = Field(+ default=None,+ description="Per-call state token budget; capped by LAYA_MAX_TOKEN_BUDGET (%d by default)"+ % DEFAULT_MAX_TOKEN_BUDGET)+ head_max_len: Optional[int] = Field(+ default=None,+ description="Per-call question-head token budget; capped by LAYA_MAX_TOKEN_BUDGET (%d "+ "by default)" % DEFAULT_MAX_TOKEN_BUDGET)+ min_confidence: Optional[float] = Field(+ default=None,+ description="Abstain when answer_confidence is below this threshold (0.0-1.0)")_v_model = field_validator("model")(classmethod(lambda cls, v: _check_model(v)))+ @model_validator(mode="before")+ @classmethod+ def _check_controls(cls, data: Any) -> Any:+ # The hook surface, the budget caps, the language-code shape and the abstention range+ # are checked by `laya.serve`'s own validators, called here on the raw body, so this+ # demo refuses the same bodies with the same 422 detail as the server it demonstrates.+ if isinstance(data, dict):+ _refuse_body_refusals(data)+ cap = _resolve_max_token_budget()+ _validate_budget_param(data, "max_len", cap)+ _validate_budget_param(data, "head_max_len", cap)+ _validate_language_param(data, "lang")+ _validate_language_param(data, "lang_guess")+ _validate_min_confidence(data)+ return data+@field_validator("state")@classmethoddef _non_empty(cls, v):@@ -211,6 +342,11 @@ class BatchRequest(BaseModel):+ _model_names())task: Optional[str] = Nonelang: Optional[str] = None+ lang_guess: Optional[str] = Field(default=None, description="See PredictRequest.lang_guess")+ max_len: Optional[int] = Field(default=None, description="See PredictRequest.max_len")+ head_max_len: Optional[int] = Field(default=None, description="See PredictRequest.head_max_len")+ min_confidence: Optional[float] = Field(default=None,+ description="See PredictRequest.min_confidence")# The batch shape, not a per-state control: these size and group the forward passes rather# than saying what a state means. `sort_by_length` only reorders a batch that is split into# more than one pass, so it needs a `batch_size` strictly between 1 and the state count.@@ -224,6 +360,22 @@ class BatchRequest(BaseModel):_v_model = field_validator("model")(classmethod(lambda cls, v: _check_model(v)))+ @model_validator(mode="before")+ @classmethod+ def _check_controls(cls, data: Any) -> Any:+ # Same four refusals and same four controls as PredictRequest -- a body that names+ # `hooks` on a batch or a `max_len` past the cap is refused here the way the shipped+ # server refuses it, not silently dropped on the way into the per-request dict.+ if isinstance(data, dict):+ _refuse_body_refusals(data)+ cap = _resolve_max_token_budget()+ _validate_budget_param(data, "max_len", cap)+ _validate_budget_param(data, "head_max_len", cap)+ _validate_language_param(data, "lang")+ _validate_language_param(data, "lang_guess")+ _validate_min_confidence(data)+ return data+# --------------------------------------------------------------------------- ## App@@ -409,9 +561,7 @@ def predict(req: PredictRequest) -> Dict[str, Any]:return _predict(req.state,_questions(req.questions),- model=req.model,- task=req.task,- lang=req.lang,+ **_per_call_controls(req),)except HTTPException:raise@@ -431,8 +581,11 @@ def predict_batch(req: BatchRequest) -> Dict[str, Any]:questions = _questions(req.questions)# Only the controls that were actually set: `route_batch` reads them with `.get`, so an# omitted key and an explicit null mean the same thing, and the request dicts stay minimal.- controls = {key: value for key, value in (("model", req.model), ("task", req.task),- ("lang", req.lang)) if value is not None}+ # `min_confidence` is the exception: `Router.predict_batch` takes it as a call argument, not+ # a per-request key, so `route_batch` would drop it from the request dict. On the batch path+ # it moves below into `shape`; on the per-state fallback below it rejoins the controls, since+ # `Router.predict` does read it as a call argument.+ controls = {k: v for k, v in _per_call_controls(req).items() if k != "min_confidence"}requests = [{"state": state, "questions": questions, **controls} for state in req.states]# The batch shape is an argument of the batch call, not a control that travels per request: the# fallback below answers one state at a time, where there is no group to size or reorder. Sent@@ -442,6 +595,8 @@ def predict_batch(req: BatchRequest) -> Dict[str, Any]:shape["batch_size"] = req.batch_sizeif req.sort_by_length:shape["sort_by_length"] = True+ if req.min_confidence is not None:+ shape["min_confidence"] = req.min_confidencetry:# One call, not one per state: `Router.predict_batch` routes the whole batch, groups it by# checkpoint and shares a forward pass across states that carry the same question schema --@@ -452,9 +607,14 @@ def predict_batch(req: BatchRequest) -> Dict[str, Any]:# its answer. Fall back to the per-state path to keep the documented envelope: N results,# with `{"index": i, "error": ...}` only where a state genuinely failed.results = []+ # On the fallback `Router.predict` takes min_confidence as a call argument, so it rejoins+ # the controls the batch path pulled out above.+ per_state = dict(controls)+ if req.min_confidence is not None:+ per_state["min_confidence"] = req.min_confidencefor i, state in enumerate(req.states):try:- results.append(_predict(state, questions, **controls))+ results.append(_predict(state, questions, **per_state))except HTTPException as item_exc: # a caller-facing status (413, 422, 503) is saferesults.append({"index": i, "error": "HTTPException: %d: %s"% (item_exc.status_code, item_exc.detail)})diff --git a/tests/test_example_server_per_call_controls.py b/tests/test_example_server_per_call_controls.pynew file mode 100644index 00000000..b9d4da5a--- /dev/null+++ b/tests/test_example_server_per_call_controls.py@@ -0,0 +1,359 @@+"""Regression: examples/server.py must forward the same per-call controls laya.serve does.++`laya/serve.py::BODY_CONTROLS` is the set of caller-side knobs the shipped `/v1/systemone`+validates and forwards to the Router -- `model`, `max_len`, `head_max_len`, `task`, `lang`,+`lang_guess`, `min_confidence` -- and it refuses `BODY_REFUSALS` (the hook surface that cannot+cross an HTTP body) rather than silently dropping it. `laya/cli.py`, `laya/mcp/tools.py` and the+`laya.integrations.*` wrappers each forward that same set.++The demo server did not. Its `PredictRequest` took only `model`/`task`/`lang`, and `BatchRequest`+added the batch shape but nothing else, so a caller who asked for a per-call token budget, a soft+language hint, or an abstention threshold got the *same* answer as one who asked for nothing. On a+deployment that runs `Router(lang_guess=...)` or `Router(min_confidence=...)`, the demo could not+override it per call; a client who asked for `max_len` past `LAYA_MAX_TOKEN_BUDGET` was served+rather than refused; and a body that named `hooks` -- the surface this file has no way to receive+-- was silently ignored instead of answered with the same 422 the shipped server gives.++Scope: what the *handlers* forward on `Router.predict` and `Router.predict_batch`, and what the+*validators* refuse before the Router is touched. The two are pinned together: a body that names+`max_len` and is refused (past the cap) never reaches the recording Router, while one that names+`max_len=256` does, and the call the handler made carries the value -- not `None`, since core+reads an absent argument as "inherit what the deployment built with" and a null would override a+Router's own `lang_guess`/`min_confidence` with the demo's default.++`min_confidence` is the one asymmetry, and it has a reason: `Router.predict_batch` takes it as a+call argument, not a per-request key (`laya/router.py:961`), so on `/predict/batch` it rides on the+call shape rather than the request dicts. The per-state fallback in that endpoint goes through+`Router.predict`, which does take it as an argument, so on the retry the same value rejoins the+controls -- the fallback is one call per state and each carries the ask.++Driven over HTTP through TestClient, on a recording stand-in at `demo.ROUTER`. No weights load.++Run: python tests/test_example_server_per_call_controls.py+"""+import os+import sys++os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")+os.environ.setdefault("LAYA_PRELOAD", "0")+os.environ.pop("LAYA_MAX_TOKEN_BUDGET", None)++ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+sys.path.insert(0, ROOT)+sys.path.insert(0, os.path.join(ROOT, "examples"))++PASS, FAIL = [], []+++def ok(name, cond, detail=""):+ (PASS if cond else FAIL).append(+ "%s%s" % (name, (" -- " + detail) if detail and not cond else ""))+ print(" %s %s%s" % ("PASS" if cond else "FAIL", name,+ (" " + detail) if detail and not cond else ""), flush=True)+++def main():+ try:+ from fastapi.testclient import TestClient+ except ImportError as exc:+ if (getattr(exc, "name", None) or "").split(".")[0] not in ("fastapi", "httpx", "starlette"):+ raise+ print("SKIP: fastapi/httpx not installed -- pip install laya[serve] httpx")+ return 0+ try:+ import server as demo+ except ImportError as exc:+ if (getattr(exc, "name", None) or "").split(".")[0] not in (+ "fastapi", "httpx", "starlette", "multipart"):+ raise+ print("SKIP: examples/server.py needs the serve extra -- pip install laya[serve]")+ return 0++ from laya.serve import BODY_CONTROLS, BODY_REFUSALS++ class RecordingRouter:+ """Answers like the Router does, and remembers how it was asked."""++ def __init__(self):+ self.predict_calls = []+ self.batch_calls = []++ def predict(self, state, questions, **kw):+ self.predict_calls.append((state, dict(kw)))+ return {"answers": {"a": {"type": "noul", "choice": False}},+ "model": "english", "state": state, "questions": questions}++ def predict_batch(self, requests, **kw):+ self.batch_calls.append((list(requests), dict(kw)))+ return [{"answers": {"a": {"type": "noul", "choice": False}},+ "model": "english", "state": r["state"], "questions": r["questions"]}+ for r in requests]++ def new_recorder():+ r = RecordingRouter()+ demo.ROUTER = r+ return r++ client = TestClient(demo.app, raise_server_exceptions=False)+ one = {"a": {"type": "noul", "instructions": "x"}}++ # --- what the two surfaces forward is the same set, drawn from laya.serve ----------+ # PredictRequest's public fields must cover BODY_CONTROLS exactly: this file's own claim+ # to be "the demo of the shipped server" fails if it stops accepting a control core takes.+ per_call_set = set(BODY_CONTROLS)+ predict_fields = set(demo.PredictRequest.model_fields) - {"state", "questions"}+ batch_fields = set(demo.BatchRequest.model_fields) - {"states", "questions",+ "batch_size", "sort_by_length"}+ ok("PredictRequest accepts every key in laya.serve.BODY_CONTROLS",+ per_call_set <= predict_fields,+ "BODY_CONTROLS=%r, PredictRequest fields=%r" % (sorted(per_call_set),+ sorted(predict_fields)))+ ok("BatchRequest accepts the same set, not a subset that drifts from the single path",+ per_call_set <= batch_fields,+ "missing from BatchRequest: %r" % sorted(per_call_set - batch_fields))+ ok("and the two models expose the same per-call surface, so a client sees no drift",+ predict_fields == batch_fields,+ "predict=%r vs batch=%r" % (sorted(predict_fields), sorted(batch_fields)))++ # --- /predict forwards each control when the caller set it -----------------------+ r = new_recorder()+ client.post("/predict", json={"state": "hello", "questions": one,+ "max_len": 256})+ ok("max_len reaches Router.predict when set",+ r.predict_calls and r.predict_calls[0][1].get("max_len") == 256,+ str(r.predict_calls[0][1] if r.predict_calls else "no call"))++ r = new_recorder()+ client.post("/predict", json={"state": "hello", "questions": one,+ "head_max_len": 64})+ ok("head_max_len reaches Router.predict when set",+ r.predict_calls and r.predict_calls[0][1].get("head_max_len") == 64)++ r = new_recorder()+ client.post("/predict", json={"state": "hello", "questions": one,+ "lang_guess": "de"})+ ok("lang_guess reaches Router.predict when set",+ r.predict_calls and r.predict_calls[0][1].get("lang_guess") == "de")++ r = new_recorder()+ client.post("/predict", json={"state": "hello", "questions": one,+ "min_confidence": 0.85})+ ok("min_confidence reaches Router.predict when set",+ r.predict_calls and r.predict_calls[0][1].get("min_confidence") == 0.85)++ r = new_recorder()+ client.post("/predict", json={"state": "hello", "questions": one,+ "model": "english", "task": "routing", "lang": "fr",+ "lang_guess": "de", "max_len": 512, "head_max_len": 32,+ "min_confidence": 0.5})+ ok("and every control on a single body arrives together, none overwriting another",+ r.predict_calls+ and r.predict_calls[0][1] == {"model": "english", "task": "routing", "lang": "fr",+ "lang_guess": "de", "max_len": 512, "head_max_len": 32,+ "min_confidence": 0.5},+ str(r.predict_calls[0][1] if r.predict_calls else "no call"))++ # --- unset controls are absent, never sent as None ------------------------------+ # `Router.predict` reads an absent argument as "inherit what the deployment built with",+ # so passing None here would override a `Router(lang_guess=...)` or a+ # `Router(min_confidence=...)` with the demo's own default.+ r = new_recorder()+ client.post("/predict", json={"state": "hello", "questions": one})+ kw = r.predict_calls[0][1] if r.predict_calls else None+ ok("an unset control is absent from the call, not sent as None",+ kw is not None and not any(kw.get(k, "__sentinel__") is None+ for k in ("max_len", "head_max_len", "lang_guess",+ "min_confidence", "task", "lang", "model")),+ str(kw))++ # `min_confidence=0.0` is falsy but a real ask -- it means "gate at zero", which is not+ # the same as "no gate". Any `if value:` guard would drop it; only `if value is not None`+ # is correct here.+ r = new_recorder()+ client.post("/predict", json={"state": "hello", "questions": one,+ "min_confidence": 0.0})+ ok("min_confidence=0.0 is still forwarded -- falsy is not absent",+ r.predict_calls and "min_confidence" in r.predict_calls[0][1]+ and r.predict_calls[0][1]["min_confidence"] == 0.0,+ str(r.predict_calls[0][1] if r.predict_calls else "no call"))++ # --- the shipped server's refusals are also refused here -----------------------+ # A body that names `hooks` cannot cross this surface: there is no receiver on the+ # demo, so a caller who asks for one today gets an answer that ignores their ask. The+ # shipped server 422s; the demo must too, on both endpoints.+ for key in BODY_REFUSALS:+ r = new_recorder()+ resp = client.post("/predict", json={"state": "hello", "questions": one, key: []})+ ok("a body that names %r is 422, not silently ignored" % key,+ resp.status_code == 422 and not r.predict_calls,+ "%s, router calls=%d" % (resp.status_code, len(r.predict_calls)))++ r = new_recorder()+ resp = client.post("/predict/batch", json={"states": ["a", "b"], "questions": one,+ "hooks_timeout": 0.5})+ ok("the refusal covers /predict/batch too",+ resp.status_code == 422 and not r.batch_calls,+ "%s, batch calls=%d" % (resp.status_code, len(r.batch_calls)))++ # --- max_len is capped by LAYA_MAX_TOKEN_BUDGET, refused before inference ------+ cap = demo._resolve_max_token_budget()+ r = new_recorder()+ resp = client.post("/predict", json={"state": "hello", "questions": one,+ "max_len": cap + 1})+ ok("a max_len past the server cap is a 422, and the Router is never asked",+ resp.status_code == 422 and not r.predict_calls,+ "%s, calls=%d" % (resp.status_code, len(r.predict_calls)))++ r = new_recorder()+ resp = client.post("/predict", json={"state": "hello", "questions": one,+ "head_max_len": cap + 1})+ ok("the same cap covers head_max_len",+ resp.status_code == 422 and not r.predict_calls)++ # The cap is read at call time, so LAYA_MAX_TOKEN_BUDGET actually moves it: the demo+ # does not carry a copy of the number.+ os.environ["LAYA_MAX_TOKEN_BUDGET"] = "2048"+ try:+ r = new_recorder()+ resp = client.post("/predict", json={"state": "hello", "questions": one,+ "max_len": 4096})+ ok("LAYA_MAX_TOKEN_BUDGET is what caps the request, not a number typed in this file",+ resp.status_code == 422 and not r.predict_calls,+ "%s" % resp.status_code)+ r = new_recorder()+ ok("and the same value under the raised cap is accepted and forwarded",+ client.post("/predict", json={"state": "hello", "questions": one,+ "max_len": 1024}).status_code == 200+ and r.predict_calls[0][1]["max_len"] == 1024)+ finally:+ os.environ.pop("LAYA_MAX_TOKEN_BUDGET", None)++ # --- type refusals, mirroring laya.serve's validators -------------------------+ # A JSON `true` where the Router expects a language code would become `"true"` through+ # core's `_english_from_code` -- a real, non-English code that decides the checkpoint.+ # `Optional[str]` in the field type also refuses a bool, so a plain status check cannot+ # tell the two paths apart; the assertion below requires laya.serve's own wording in+ # the 422 body, which only fires when this file actually calls `_validate_language_param`.+ for key in ("lang", "lang_guess"):+ r = new_recorder()+ resp = client.post("/predict", json={"state": "hello", "questions": one, key: True})+ detail = str(resp.json().get("detail", ""))+ ok("%s: a JSON bool is refused before the Router is asked" % key,+ resp.status_code == 422 and not r.predict_calls,+ "%s, calls=%d" % (resp.status_code, len(r.predict_calls)))+ ok("%s: the 422 detail is laya.serve's own wording, not pydantic's field error" % key,+ "language code string" in detail, detail[:180])++ r = new_recorder()+ resp = client.post("/predict", json={"state": "hello", "questions": one,+ "min_confidence": True})+ ok("min_confidence: a JSON bool is refused -- it is not a threshold",+ resp.status_code == 422 and not r.predict_calls,+ "%s, calls=%d" % (resp.status_code, len(r.predict_calls)))++ r = new_recorder()+ resp = client.post("/predict", json={"state": "hello", "questions": one,+ "min_confidence": 1.5})+ ok("min_confidence outside core's [0, 1] is refused before an inference slot",+ resp.status_code == 422 and not r.predict_calls)++ r = new_recorder()+ resp = client.post("/predict", json={"state": "hello", "questions": one,+ "max_len": "abc"})+ ok("a non-integer max_len is refused, not coerced",+ resp.status_code == 422 and not r.predict_calls)++ r = new_recorder()+ resp = client.post("/predict", json={"state": "hello", "questions": one,+ "max_len": -1})+ ok("a non-positive max_len is refused, matching laya.serve._validate_budget_param",+ resp.status_code == 422 and not r.predict_calls)++ # --- /predict/batch forwards the same set -------------------------------------+ # On the batch path the per-request controls ride in each request dict (so `route_batch`+ # and `predict_batch` read them per state), while `min_confidence` is a call-level+ # argument to `Router.predict_batch` -- see laya/router.py:961.+ r = new_recorder()+ states = ["s1", "s2", "s3"]+ client.post("/predict/batch", json={"states": states, "questions": one,+ "lang": "fr", "lang_guess": "de",+ "max_len": 512, "head_max_len": 64,+ "min_confidence": 0.75,+ "batch_size": 2, "sort_by_length": True})+ reqs, shape = r.batch_calls[0] if r.batch_calls else ([], {})+ ok("each request on the batch carries the caller's max_len/head_max_len/lang/lang_guess",+ all(rr.get("max_len") == 512 and rr.get("head_max_len") == 64+ and rr.get("lang") == "fr" and rr.get("lang_guess") == "de" for rr in reqs),+ str(reqs[:1]))+ ok("min_confidence rides on the batch call as a shape argument, not per request",+ shape.get("min_confidence") == 0.75+ and not any("min_confidence" in rr for rr in reqs),+ "shape=%r" % shape)+ ok("batch_size and sort_by_length still reach the Router alongside it",+ shape.get("batch_size") == 2 and shape.get("sort_by_length") is True)++ # Unset controls on the batch path stay absent too, for the same "inherit" reason.+ r = new_recorder()+ client.post("/predict/batch", json={"states": states, "questions": one})+ reqs, shape = r.batch_calls[0] if r.batch_calls else ([], {})+ ok("a plain batch sends no null controls and no shape",+ reqs and not any(k in rr for rr in reqs for k in+ ("max_len", "head_max_len", "lang", "lang_guess", "min_confidence",+ "task", "model"))+ and not shape,+ "reqs=%r shape=%r" % (reqs[:1], shape))++ # --- the fallback keeps the ask on every per-state retry ----------------------+ # When Router.predict_batch fails, the endpoint retries each state through Router.predict,+ # which *does* take min_confidence as a call argument. If the retry dropped it, one bad+ # state would silently cost the neighbours their abstention gate.+ class HalfRouter(RecordingRouter):+ def predict_batch(self, requests, **kw):+ self.batch_calls.append((list(requests), dict(kw)))+ raise ValueError("batch forward failed")++ r = HalfRouter()+ demo.ROUTER = r+ body = client.post("/predict/batch", json={"states": states, "questions": one,+ "max_len": 256, "min_confidence": 0.75}).json()+ ok("the fallback retried every state",+ len(body.get("results", [])) == 3 and len(r.predict_calls) == 3)+ ok("and each retry carries the caller's max_len",+ all(c[1].get("max_len") == 256 for c in r.predict_calls),+ str([c[1] for c in r.predict_calls][:1]))+ ok("and each retry carries the abstention threshold, this time as a call argument",+ all(c[1].get("min_confidence") == 0.75 for c in r.predict_calls),+ str([c[1] for c in r.predict_calls][:1]))+ # batch_size and sort_by_length belong to the forward-pass shape only; the fallback is one+ # state per call, so leaking them into `Router.predict(...)` would TypeError every retry.+ ok("and the fallback does not leak batch shape into Router.predict",+ not any("batch_size" in c[1] or "sort_by_length" in c[1] for c in r.predict_calls),+ str([c[1] for c in r.predict_calls][:1]))++ # --- parity with laya.serve's refusal wording ---------------------------------+ # The detail must be the shipped server's, not a paraphrase: a client that reads the+ # 422 body gets the same "install them where laya-serve runs" guidance from both.+ resp = client.post("/predict", json={"state": "hello", "questions": one, "hooks": []})+ ok("the 422 detail is laya.serve's own wording, not a copy",+ resp.status_code == 422+ and "run inside the server process" in str(resp.json().get("detail", "")),+ str(resp.json())[:220])++ # --- the two endpoints refuse identically -------------------------------------+ # A client who POSTs `{"hooks_timeout": 0.5}` to the batch endpoint must not get a+ # different answer than one who POSTs the same field to the single endpoint.+ for key in BODY_REFUSALS:+ single = client.post("/predict", json={"state": "hi", "questions": one, key: 1})+ batch = client.post("/predict/batch", json={"states": ["a"], "questions": one, key: 1})+ ok("%s refused the same way on both endpoints" % key,+ single.status_code == batch.status_code == 422,+ "single=%s batch=%s" % (single.status_code, batch.status_code))++ demo.ROUTER = None+ print()+ print("%d passed, %d failed" % (len(PASS), len(FAIL)))+ return 0 if not FAIL else 1+++if __name__ == "__main__":+ sys.exit(main())