Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(mcp): carry each batch request's token budget to the forward pass
Aashishd613aeabb44c
diff --git a/README.md b/README.mdindex 4aadb0c4..7b491aa8 100644--- a/README.md+++ b/README.md@@ -935,9 +935,11 @@ package:The tools return structured JSON (answers with probabilities, routing metadata, device,`latency_ms`). `laya_predict_batch` and `laya_route_batch` are the MCP form of[`Router.predict_batch` / `route_batch`](#batch-mode-score-many-states-in-one-forward-pass):-one tool call takes an array of `{state, questions, model?, lang?}` requests, routes them+one tool call takes an array of `{state, questions, model?, task?, lang?, lang_guess?, max_len?, head_max_len?}` requests, routes themfirst, groups them by checkpoint, and shares forward passes between requests with the same-question schema, returning the answers in input order. On 16 mixed-language tickets through+question schema, returning the answers in input order. `max_len` / `head_max_len` are per request+there, exactly as `Router.predict_batch` reads them, so one wide question can raise its own budget+without shrinking the rest of the batch to it. On 16 mixed-language tickets throughthe tool functions themselves, one batch call beat 16 `laya_predict` calls by **2.1-2.3x onMPS** (983-1082 ms -> 467-477 ms) and **~1.25x on CPU** (1861-2471 ms -> 1470-1911 ms), with**0/16 decision flips** (choice label, rounded score, noul sign) against the loop. Prefer itdiff --git a/laya/mcp/server.py b/laya/mcp/server.pyindex 9e3342e4..b2bed294 100644--- a/laya/mcp/server.py+++ b/laya/mcp/server.py@@ -42,6 +42,7 @@ from laya.serve import _apply_thread_limit, _env_boolfrom .device import env_devicefrom .tools import (ToolError,+ batch_item_key_doc,laya_decide,get_available_presets,laya_predict,@@ -268,10 +269,13 @@ def laya_predict_tool(name="laya_predict_batch",description=("Answer many typed-question requests in one call: requests is a non-empty array of "- "{state, questions, model?, task?, lang?, lang_guess?} objects, each with the same "+ + batch_item_key_doc() + ", each with the same ""questions schema as laya_predict. Requests are routed first, grouped by checkpoint, ""and share forward passes when their question schemas match, so scoring many "- "requests costs one round trip instead of N. Returns answers in input order with "+ "requests costs one round trip instead of N. max_len / head_max_len are per request "+ "here, not per call: a wide question can raise its own budget without shrinking the "+ "batch's other requests to it, and requests that ask for different budgets are split "+ "into separate forward passes. Returns answers in input order with ""per-request routing and device, plus model_counts and batch latency. "+ _GUARDRAILS),@@ -294,7 +298,7 @@ def laya_predict_batch_tool(requests: list, batch_size: int = 0) -> str:description=("Decide which Laya checkpoint would answer each request, without running any ""forward pass or loading a checkpoint: requests is a non-empty array of "- "{state, questions, model?, task?, lang?, lang_guess?} objects. Use this to "+ + batch_item_key_doc(omit=("max_len", "head_max_len")) + ". Use this to ""inspect or aggregate the routing of a workload before paying model-load cost. ""Returns one {model, repo, reason} decision per request in input order, plus ""model_counts. "diff --git a/laya/mcp/tools.py b/laya/mcp/tools.pyindex ee294383..5484602c 100644--- a/laya/mcp/tools.py+++ b/laya/mcp/tools.py@@ -792,6 +792,20 @@ def _validate_batch_str(value: Any, where: str, allow_empty: bool = False) -> streturn value+def _validate_batch_budget(value: Any, name: str, where: str) -> int | None:+ """One of the two per-request token budgets, with the batch item named in the message.++ Same guard and same error code as the single-request tools take, so `invalid_max_len` means+ one thing whichever tool a client called.+ """+ if value is None:+ return None+ try:+ return validate_budget(value, name)+ except ToolError as error:+ raise ToolError("invalid_%s" % name, "%s[%r]: %s" % (where, name, error)) from None++def _validate_batch_item(request: Any, i: int) -> dict:where = "requests[%d]" % iif not isinstance(request, dict):@@ -809,20 +823,44 @@ def _validate_batch_item(request: Any, i: int) -> dict:item[key] = _validate_batch_str(request[key], "%s[%r]" % (where, key))if "lang_guess" in request:item["lang_guess"] = request["lang_guess"]+ # `Router.predict_batch` reads both off the request dict and splits requests that ask for+ # different budgets into separate forward passes, so an item that names one must keep it.+ for key in ("max_len", "head_max_len"):+ if key in request:+ budget = _validate_batch_budget(request[key], key, where)+ if budget is not None:+ item[key] = budgetreturn item+# The per-request overrides a batch item may carry. Written out from here rather than retyped,+# because the key list a client sees is the key list `_validate_batch_item` keeps: an enumeration+# that drops one advertises a control that would be ignored, and an enumeration that gains one+# rejects a request the tool would have answered.+BATCH_ITEM_OVERRIDES = ("model", "task", "lang", "lang_guess", "max_len", "head_max_len")+++def batch_item_key_doc(omit: tuple = ()) -> str:+ """The batch item shape as one phrase: ``{state, questions, model?, task?, ...}``.++ `omit` drops the overrides a given tool has no use for -- the token budgets are kept by the+ shared validator but mean nothing to `route_batch`, which never runs a forward pass.+ """+ keys = [key for key in BATCH_ITEM_OVERRIDES if key not in omit]+ return "{state, questions, %s} objects" % ", ".join("%s?" % key for key in keys)++def validate_batch_requests(requests: Any) -> list[dict]:"""Validate a tool payload of many requests, preserving order.Same per-item validation as ``laya_predict``/``laya_route`` (state, questions,- optional model/task/lang/lang_guess overrides), run before any model loads so- one malformed item fails the whole call instead of a partial batch.+ optional model/task/lang/lang_guess/max_len/head_max_len overrides), run before any model+ loads so one malformed item fails the whole call instead of a partial batch."""if not isinstance(requests, list) or not requests:raise ToolError("invalid_request",- "requests must be a non-empty array of {state, questions, model?, task?, lang?, lang_guess?} objects",+ "requests must be a non-empty array of %s" % batch_item_key_doc(),)return [_validate_batch_item(request, i) for i, request in enumerate(requests)]diff --git a/tests/test_mcp.py b/tests/test_mcp.pyindex ce228995..072f6f94 100644--- a/tests/test_mcp.py+++ b/tests/test_mcp.py@@ -28,10 +28,12 @@ except ImportError:from laya.mcp.device import agent_device, device_report, env_device, resolve_device, router_agent # noqa: E402from laya.mcp.server import _models_from_env, server as mcp_server # noqa: E402from laya.mcp.tools import ( # noqa: E402+ BATCH_ITEM_OVERRIDES,PRESETS,PRESET_ALIASES,ToolError,_overrides,+ batch_item_key_doc,get_available_presets,laya_decide,laya_predict,@@ -891,6 +893,37 @@ def test_batch_validation():# Keys the Router would never expect are dropped, not forwarded.out = validate_batch_requests([{"state": STATE, "questions": QUESTIONS, "temperature": 0}])ok("batch/validation_unknown_key_dropped", set(out[0]) == {"state", "questions"}, repr(out[0]))+ # The two token budgets are per-request overrides Router.predict_batch reads off the item,+ # so they survive validation rather than being dropped with the unknown keys above.+ out = validate_batch_requests([{"state": STATE, "questions": QUESTIONS,+ "max_len": 1024, "head_max_len": 512}])+ ok("batch/validation_keeps_budgets", out[0].get("max_len") == 1024+ and out[0].get("head_max_len") == 512, repr(out[0]))+ # An explicit null means "keep the checkpoint's default", the same as leaving it out.+ out = validate_batch_requests([{"state": STATE, "questions": QUESTIONS, "max_len": None}])+ ok("batch/validation_null_budget_dropped", "max_len" not in out[0], repr(out[0]))+ for key, bad in (("max_len", 0), ("max_len", "1024"), ("max_len", 2.5), ("max_len", True),+ ("head_max_len", 0), ("head_max_len", -1), ("head_max_len", "512")):+ expect_tool_error("batch/item_bad_%s_%r" % (key, bad),+ lambda k=key, v=bad: validate_batch_requests(+ [{"state": STATE, "questions": QUESTIONS, k: v}]),+ "invalid_%s" % key)+ # The message that lists the accepted keys must list the ones the validator keeps --+ # a client reads that list as the schema. Both directions, derived from the validator.+ probe = {"state": STATE, "questions": QUESTIONS, "model": "english", "task": "massive",+ "lang": "en", "lang_guess": False, "max_len": 5, "head_max_len": 5}+ kept = set(validate_batch_requests([probe])[0])+ ok("batch/validation_covers_every_override",+ kept == {"state", "questions"} | set(BATCH_ITEM_OVERRIDES), repr(sorted(kept)))+ message = ""+ try:+ validate_batch_requests([])+ except ToolError as exc:+ message = str(exc)+ listed = {part.strip().rstrip("?") for part in+ message[message.index("{") + 1:message.index("}")].split(",")}+ ok("batch/error_message_matches_validator", listed == kept,+ "listed=%r kept=%r" % (sorted(listed), sorted(kept)))def test_batch_predict():@@ -920,6 +953,21 @@ def test_batch_predict():ok("batch/predict_size_forwarded", size == 8, repr(size))ok("batch/predict_items_forwarded", [item["state"] for item in forwarded]== [request["state"] for request in BATCH_REQUESTS])+ # A request's own token budget must reach the forward pass: `Router.predict_batch` reads+ # max_len/head_max_len off each item and splits items that ask for different budgets into+ # separate calls, so an item that loses them is answered at the checkpoint's default window.+ router = BatchRouter()+ laya_predict_batch([{"state": STATE, "questions": QUESTIONS,+ "max_len": 1024, "head_max_len": 512},+ {"state": {"body": "no budget asked for"}, "questions": QUESTIONS}],+ router=router)+ forwarded = router.predict_batch_calls[0][0]+ ok("batch/predict_budget_forwarded",+ forwarded[0].get("max_len") == 1024 and forwarded[0].get("head_max_len") == 512,+ repr({k: v for k, v in forwarded[0].items() if k not in ("state", "questions")}))+ ok("batch/predict_budget_absent_when_unset",+ "max_len" not in forwarded[1] and "head_max_len" not in forwarded[1],+ repr(sorted(forwarded[1])))ok("batch/predict_keys", set(out) == {"requests", "model_counts","total_latency_ms", "per_request_latency_ms"}, repr(sorted(out)))@@ -1751,6 +1799,34 @@ def test_timeout_removed():# --- server registration (schema only, no model load) ------------------------+def test_batch_item_shape_as_documented():+ """Every place the batch item shape is written out says what the validator keeps.++ Three strings enumerate the keys a batch item may carry: `validate_batch_requests`' error+ message (checked against the validator in test_batch_validation), each batch tool's registered+ description, and the README's MCP section. A client reads them as the schema, so all three are+ compared to `BATCH_ITEM_OVERRIDES` -- the tuple `_validate_batch_item` branches on.+ `laya_route_batch` leaves the two token budgets out, because routing runs no forward pass and+ a budget there would be ignored.+ """+ by_name = {t.name: t for t in asyncio.run(mcp_server.list_tools())}+ for tool, omit in (("laya_predict_batch", ()),+ ("laya_route_batch", ("max_len", "head_max_len"))):+ want = batch_item_key_doc(omit=omit)+ ok("schema/%s_item_shape" % tool, want in (by_name[tool].description or ""),+ "documents %r" % want)++ readme = (Path(__file__).resolve().parents[1] / "README.md").read_text()+ marker = "one tool call takes an array of `{"+ ok("docs/readme_names_the_batch_shape", marker in readme, "sentence not found")+ if marker in readme:+ start = readme.index(marker) + len(marker)+ inner = readme[start:readme.index("}", start)]+ named = {part.strip().rstrip("?") for part in inner.split(",")}+ ok("docs/readme_batch_item_keys",+ named == {"state", "questions"} | set(BATCH_ITEM_OVERRIDES), repr(sorted(named)))++def test_models_from_env():old = os.environ.get("LAYA_MODELS")try:@@ -1884,6 +1960,7 @@ test_question_validation_matches_the_agent()test_a_bad_question_is_a_caller_error_not_a_server_fault()test_timeout_removed()test_models_from_env()+test_batch_item_shape_as_documented()test_auto_task_env()test_server_registration()