Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(mcp): carry each batch request's token budget to the forward pass

Aashishd613aeabb44c
diff --git a/README.md b/README.md
index 4aadb0c4..7b491aa8 100644
--- a/README.md
+++ b/README.md
@@ -935,9 +935,11 @@ package:
The tools return structured JSON (answers with probabilities, routing metadata, device,
`latency_ms`). `laya_predict_batch` and `laya_route_batch` are the MCP form of
[`Router.predict_batch` / `route_batch`](#batch-mode-score-many-states-in-one-forward-pass):
-one tool call takes an array of `{state, questions, model?, lang?}` requests, routes them
+one tool call takes an array of `{state, questions, model?, task?, lang?, lang_guess?, max_len?, head_max_len?}` requests, routes them
first, groups them by checkpoint, and shares forward passes between requests with the same
-question schema, returning the answers in input order. On 16 mixed-language tickets through
+question schema, returning the answers in input order. `max_len` / `head_max_len` are per request
+there, exactly as `Router.predict_batch` reads them, so one wide question can raise its own budget
+without shrinking the rest of the batch to it. On 16 mixed-language tickets through
the tool functions themselves, one batch call beat 16 `laya_predict` calls by **2.1-2.3x on
MPS** (983-1082 ms -> 467-477 ms) and **~1.25x on CPU** (1861-2471 ms -> 1470-1911 ms), with
**0/16 decision flips** (choice label, rounded score, noul sign) against the loop. Prefer it
diff --git a/laya/mcp/server.py b/laya/mcp/server.py
index 9e3342e4..b2bed294 100644
--- a/laya/mcp/server.py
+++ b/laya/mcp/server.py
@@ -42,6 +42,7 @@ from laya.serve import _apply_thread_limit, _env_bool
from .device import env_device
from .tools import (
ToolError,
+ batch_item_key_doc,
laya_decide,
get_available_presets,
laya_predict,
@@ -268,10 +269,13 @@ def laya_predict_tool(
name="laya_predict_batch",
description=(
"Answer many typed-question requests in one call: requests is a non-empty array of "
- "{state, questions, model?, task?, lang?, lang_guess?} objects, each with the same "
+ + batch_item_key_doc() + ", each with the same "
"questions schema as laya_predict. Requests are routed first, grouped by checkpoint, "
"and share forward passes when their question schemas match, so scoring many "
- "requests costs one round trip instead of N. Returns answers in input order with "
+ "requests costs one round trip instead of N. max_len / head_max_len are per request "
+ "here, not per call: a wide question can raise its own budget without shrinking the "
+ "batch's other requests to it, and requests that ask for different budgets are split "
+ "into separate forward passes. Returns answers in input order with "
"per-request routing and device, plus model_counts and batch latency. "
+ _GUARDRAILS
),
@@ -294,7 +298,7 @@ def laya_predict_batch_tool(requests: list, batch_size: int = 0) -> str:
description=(
"Decide which Laya checkpoint would answer each request, without running any "
"forward pass or loading a checkpoint: requests is a non-empty array of "
- "{state, questions, model?, task?, lang?, lang_guess?} objects. Use this to "
+ + batch_item_key_doc(omit=("max_len", "head_max_len")) + ". Use this to "
"inspect or aggregate the routing of a workload before paying model-load cost. "
"Returns one {model, repo, reason} decision per request in input order, plus "
"model_counts. "
diff --git a/laya/mcp/tools.py b/laya/mcp/tools.py
index ee294383..5484602c 100644
--- a/laya/mcp/tools.py
+++ b/laya/mcp/tools.py
@@ -792,6 +792,20 @@ def _validate_batch_str(value: Any, where: str, allow_empty: bool = False) -> st
return value
+def _validate_batch_budget(value: Any, name: str, where: str) -> int | None:
+ """One of the two per-request token budgets, with the batch item named in the message.
+
+ Same guard and same error code as the single-request tools take, so `invalid_max_len` means
+ one thing whichever tool a client called.
+ """
+ if value is None:
+ return None
+ try:
+ return validate_budget(value, name)
+ except ToolError as error:
+ raise ToolError("invalid_%s" % name, "%s[%r]: %s" % (where, name, error)) from None
+
+
def _validate_batch_item(request: Any, i: int) -> dict:
where = "requests[%d]" % i
if not isinstance(request, dict):
@@ -809,20 +823,44 @@ def _validate_batch_item(request: Any, i: int) -> dict:
item[key] = _validate_batch_str(request[key], "%s[%r]" % (where, key))
if "lang_guess" in request:
item["lang_guess"] = request["lang_guess"]
+ # `Router.predict_batch` reads both off the request dict and splits requests that ask for
+ # different budgets into separate forward passes, so an item that names one must keep it.
+ for key in ("max_len", "head_max_len"):
+ if key in request:
+ budget = _validate_batch_budget(request[key], key, where)
+ if budget is not None:
+ item[key] = budget
return item
+# The per-request overrides a batch item may carry. Written out from here rather than retyped,
+# because the key list a client sees is the key list `_validate_batch_item` keeps: an enumeration
+# that drops one advertises a control that would be ignored, and an enumeration that gains one
+# rejects a request the tool would have answered.
+BATCH_ITEM_OVERRIDES = ("model", "task", "lang", "lang_guess", "max_len", "head_max_len")
+
+
+def batch_item_key_doc(omit: tuple = ()) -> str:
+ """The batch item shape as one phrase: ``{state, questions, model?, task?, ...}``.
+
+ `omit` drops the overrides a given tool has no use for -- the token budgets are kept by the
+ shared validator but mean nothing to `route_batch`, which never runs a forward pass.
+ """
+ keys = [key for key in BATCH_ITEM_OVERRIDES if key not in omit]
+ return "{state, questions, %s} objects" % ", ".join("%s?" % key for key in keys)
+
+
def validate_batch_requests(requests: Any) -> list[dict]:
"""Validate a tool payload of many requests, preserving order.
Same per-item validation as ``laya_predict``/``laya_route`` (state, questions,
- optional model/task/lang/lang_guess overrides), run before any model loads so
- one malformed item fails the whole call instead of a partial batch.
+ optional model/task/lang/lang_guess/max_len/head_max_len overrides), run before any model
+ loads so one malformed item fails the whole call instead of a partial batch.
"""
if not isinstance(requests, list) or not requests:
raise ToolError(
"invalid_request",
- "requests must be a non-empty array of {state, questions, model?, task?, lang?, lang_guess?} objects",
+ "requests must be a non-empty array of %s" % batch_item_key_doc(),
)
return [_validate_batch_item(request, i) for i, request in enumerate(requests)]
diff --git a/tests/test_mcp.py b/tests/test_mcp.py
index ce228995..072f6f94 100644
--- a/tests/test_mcp.py
+++ b/tests/test_mcp.py
@@ -28,10 +28,12 @@ except ImportError:
from laya.mcp.device import agent_device, device_report, env_device, resolve_device, router_agent # noqa: E402
from laya.mcp.server import _models_from_env, server as mcp_server # noqa: E402
from laya.mcp.tools import ( # noqa: E402
+ BATCH_ITEM_OVERRIDES,
PRESETS,
PRESET_ALIASES,
ToolError,
_overrides,
+ batch_item_key_doc,
get_available_presets,
laya_decide,
laya_predict,
@@ -891,6 +893,37 @@ def test_batch_validation():
# Keys the Router would never expect are dropped, not forwarded.
out = validate_batch_requests([{"state": STATE, "questions": QUESTIONS, "temperature": 0}])
ok("batch/validation_unknown_key_dropped", set(out[0]) == {"state", "questions"}, repr(out[0]))
+ # The two token budgets are per-request overrides Router.predict_batch reads off the item,
+ # so they survive validation rather than being dropped with the unknown keys above.
+ out = validate_batch_requests([{"state": STATE, "questions": QUESTIONS,
+ "max_len": 1024, "head_max_len": 512}])
+ ok("batch/validation_keeps_budgets", out[0].get("max_len") == 1024
+ and out[0].get("head_max_len") == 512, repr(out[0]))
+ # An explicit null means "keep the checkpoint's default", the same as leaving it out.
+ out = validate_batch_requests([{"state": STATE, "questions": QUESTIONS, "max_len": None}])
+ ok("batch/validation_null_budget_dropped", "max_len" not in out[0], repr(out[0]))
+ for key, bad in (("max_len", 0), ("max_len", "1024"), ("max_len", 2.5), ("max_len", True),
+ ("head_max_len", 0), ("head_max_len", -1), ("head_max_len", "512")):
+ expect_tool_error("batch/item_bad_%s_%r" % (key, bad),
+ lambda k=key, v=bad: validate_batch_requests(
+ [{"state": STATE, "questions": QUESTIONS, k: v}]),
+ "invalid_%s" % key)
+ # The message that lists the accepted keys must list the ones the validator keeps --
+ # a client reads that list as the schema. Both directions, derived from the validator.
+ probe = {"state": STATE, "questions": QUESTIONS, "model": "english", "task": "massive",
+ "lang": "en", "lang_guess": False, "max_len": 5, "head_max_len": 5}
+ kept = set(validate_batch_requests([probe])[0])
+ ok("batch/validation_covers_every_override",
+ kept == {"state", "questions"} | set(BATCH_ITEM_OVERRIDES), repr(sorted(kept)))
+ message = ""
+ try:
+ validate_batch_requests([])
+ except ToolError as exc:
+ message = str(exc)
+ listed = {part.strip().rstrip("?") for part in
+ message[message.index("{") + 1:message.index("}")].split(",")}
+ ok("batch/error_message_matches_validator", listed == kept,
+ "listed=%r kept=%r" % (sorted(listed), sorted(kept)))
def test_batch_predict():
@@ -920,6 +953,21 @@ def test_batch_predict():
ok("batch/predict_size_forwarded", size == 8, repr(size))
ok("batch/predict_items_forwarded", [item["state"] for item in forwarded]
== [request["state"] for request in BATCH_REQUESTS])
+ # A request's own token budget must reach the forward pass: `Router.predict_batch` reads
+ # max_len/head_max_len off each item and splits items that ask for different budgets into
+ # separate calls, so an item that loses them is answered at the checkpoint's default window.
+ router = BatchRouter()
+ laya_predict_batch([{"state": STATE, "questions": QUESTIONS,
+ "max_len": 1024, "head_max_len": 512},
+ {"state": {"body": "no budget asked for"}, "questions": QUESTIONS}],
+ router=router)
+ forwarded = router.predict_batch_calls[0][0]
+ ok("batch/predict_budget_forwarded",
+ forwarded[0].get("max_len") == 1024 and forwarded[0].get("head_max_len") == 512,
+ repr({k: v for k, v in forwarded[0].items() if k not in ("state", "questions")}))
+ ok("batch/predict_budget_absent_when_unset",
+ "max_len" not in forwarded[1] and "head_max_len" not in forwarded[1],
+ repr(sorted(forwarded[1])))
ok("batch/predict_keys", set(out) == {"requests", "model_counts",
"total_latency_ms", "per_request_latency_ms"}, repr(sorted(out)))
@@ -1751,6 +1799,34 @@ def test_timeout_removed():
# --- server registration (schema only, no model load) ------------------------
+def test_batch_item_shape_as_documented():
+ """Every place the batch item shape is written out says what the validator keeps.
+
+ Three strings enumerate the keys a batch item may carry: `validate_batch_requests`' error
+ message (checked against the validator in test_batch_validation), each batch tool's registered
+ description, and the README's MCP section. A client reads them as the schema, so all three are
+ compared to `BATCH_ITEM_OVERRIDES` -- the tuple `_validate_batch_item` branches on.
+ `laya_route_batch` leaves the two token budgets out, because routing runs no forward pass and
+ a budget there would be ignored.
+ """
+ by_name = {t.name: t for t in asyncio.run(mcp_server.list_tools())}
+ for tool, omit in (("laya_predict_batch", ()),
+ ("laya_route_batch", ("max_len", "head_max_len"))):
+ want = batch_item_key_doc(omit=omit)
+ ok("schema/%s_item_shape" % tool, want in (by_name[tool].description or ""),
+ "documents %r" % want)
+
+ readme = (Path(__file__).resolve().parents[1] / "README.md").read_text()
+ marker = "one tool call takes an array of `{"
+ ok("docs/readme_names_the_batch_shape", marker in readme, "sentence not found")
+ if marker in readme:
+ start = readme.index(marker) + len(marker)
+ inner = readme[start:readme.index("}", start)]
+ named = {part.strip().rstrip("?") for part in inner.split(",")}
+ ok("docs/readme_batch_item_keys",
+ named == {"state", "questions"} | set(BATCH_ITEM_OVERRIDES), repr(sorted(named)))
+
+
def test_models_from_env():
old = os.environ.get("LAYA_MODELS")
try:
@@ -1884,6 +1960,7 @@ test_question_validation_matches_the_agent()
test_a_bad_question_is_a_caller_error_not_a_server_fault()
test_timeout_removed()
test_models_from_env()
+test_batch_item_shape_as_documented()
test_auto_task_env()
test_server_registration()