Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(mcp): add laya_predict_batch and laya_route_batch tools

Aashishd52056bc6491
diff --git a/laya/mcp/server.py b/laya/mcp/server.py
index 0618af61..77debc0f 100644
--- a/laya/mcp/server.py
+++ b/laya/mcp/server.py
@@ -40,8 +40,10 @@ from .device import env_device
from .tools import (
ToolError,
laya_predict,
+ laya_predict_batch,
laya_preset,
laya_route,
+ laya_route_batch,
laya_shortlist,
laya_status,
)
@@ -63,8 +65,8 @@ _ROUTER_LOCK = threading.Lock()
# two, and auto-routing never selects it).
_DEFAULT_MODELS = ("english", "multilingual")
-# Shared usage guardrails for the three decision tools. laya_status reports
-# instead of deciding, so it does not carry them.
+# Shared usage guardrails for the decision tools. laya_status reports instead
+# of deciding, so it does not carry them.
_GUARDRAILS = (
"Use for structured decisions only: choice (finite labels), score (ordinal rubric), "
"noul (calibrated P(true)). One forward pass ~33ms (GPU) / ~200ms (CPU). "
@@ -199,6 +201,49 @@ def laya_predict_tool(state: dict, questions: dict, model: str = "auto") -> str:
)
+ name="laya_predict_batch",
+ description=(
+ "Answer many typed-question requests in one call: requests is a non-empty array of "
+ "{state, questions, model?, task?, lang?, lang_guess?} objects, each with the same "
+ "questions schema as laya_predict. Requests are routed first, grouped by checkpoint, "
+ "and share forward passes when their question schemas match, so scoring many "
+ "requests costs one round trip instead of N. Returns answers in input order with "
+ "per-request routing and device, plus model_counts and batch latency. "
+ + _GUARDRAILS
+ ),
+)
+def laya_predict_batch_tool(requests: list, batch_size: int = 0) -> str:
+ """Answer many typed-question requests in one batched call."""
+ router = _router_or_error()
+ # batch_size=0 means "unset": MCP clients send defaults eagerly, and
+ # None is what Router.predict_batch takes as "no forward-pass cap".
+ return _wrap(
+ laya_predict_batch,
+ requests=requests,
+ batch_size=batch_size or None,
+ router=router,
+ )
+
+
+ name="laya_route_batch",
+ description=(
+ "Decide which Laya checkpoint would answer each request, without running any "
+ "forward pass or loading a checkpoint: requests is a non-empty array of "
+ "{state, questions, model?, task?, lang?, lang_guess?} objects. Use this to "
+ "inspect or aggregate the routing of a workload before paying model-load cost. "
+ "Returns one {model, repo, reason} decision per request in input order, plus "
+ "model_counts. "
+ + _GUARDRAILS
+ ),
+)
+def laya_route_batch_tool(requests: list) -> str:
+ """Route many requests to checkpoints without a forward pass."""
+ router = _router_or_error()
+ return _wrap(laya_route_batch, requests=requests, router=router)
+
+
@server.tool(
name="laya_shortlist",
description=(
diff --git a/laya/mcp/tools.py b/laya/mcp/tools.py
index 810f5edb..ca117b90 100644
--- a/laya/mcp/tools.py
+++ b/laya/mcp/tools.py
@@ -117,6 +117,31 @@ def _normalize_answers(raw: Any) -> dict:
return raw
+def _normalize_result(result: Any) -> dict:
+ if not isinstance(result, dict):
+ raise ToolError("internal_error", "predict returned non-object")
+ # Router.predict and Agent.system_one both return the system_one payload,
+ # which always carries an "answers" object (empty for empty questions).
+ try:
+ answers = result["answers"]
+ except KeyError:
+ raise ToolError("internal_error", "predict result has no 'answers' object") from None
+ return {"answers": _normalize_answers(answers), "routing": result.get("routing")}
+
+
+def _reading_device(router: Any, agent: Any, model_name: str, routing: dict | None) -> str | None:
+ """Real device of the checkpoint that answered: Agent.device reflects a
+ silent GPU -> CPU fallback. Omitted when it cannot be read, rather than
+ guessed.
+ """
+ if model_name != "auto" and agent is not None:
+ return agent_device(agent)
+ model_used = (routing or {}).get("model")
+ if isinstance(model_used, str) and model_used:
+ return agent_device(router_agent(router, model_used))
+ return None
+
+
def laya_predict(
state: Any,
questions: Any,
@@ -148,22 +173,10 @@ def laya_predict(
result = _run()
latency_ms = (time.perf_counter() - started) * 1000.0
- if not isinstance(result, dict):
- raise ToolError("internal_error", "predict returned non-object")
- # Router.predict and Agent.system_one both return the system_one payload,
- # which always carries an "answers" object (empty for empty questions).
- answers = _normalize_answers(result["answers"])
- routing = result.get("routing") or {"model": model_name, "repo": None, "reason": "explicit model"}
- # Real device of the checkpoint that answered: Agent.device reflects a
- # silent GPU -> CPU fallback. Omitted when it cannot be read, rather than
- # guessed.
- device = None
- if model_name != "auto" and agent is not None:
- device = agent_device(agent)
- else:
- model_used = routing.get("model")
- if isinstance(model_used, str) and model_used:
- device = agent_device(router_agent(router, model_used))
+ norm = _normalize_result(result)
+ answers = norm["answers"]
+ routing = norm["routing"] or {"model": model_name, "repo": None, "reason": "explicit model"}
+ device = _reading_device(router, agent, model_name, norm["routing"])
out: dict[str, Any] = {
"answers": answers,
"routing": routing,
@@ -174,15 +187,7 @@ def laya_predict(
return out
-def laya_route(state: Any, questions: Any, *, router: Any = None) -> dict:
- """Routing decision only: no forward pass."""
- state_d = validate_state(state)
- questions_d = validate_questions(questions)
- if router is None:
- raise ToolError("models_not_ready", "Router is not loaded")
- if not hasattr(router, "route"):
- raise ToolError("internal_error", "router has no route() method")
- decision = router.route(state_d, questions_d)
+def _decision_to_dict(decision: Any) -> dict:
if isinstance(decision, dict):
return {
"model": decision.get("model"),
@@ -196,6 +201,17 @@ def laya_route(state: Any, questions: Any, *, router: Any = None) -> dict:
}
+def laya_route(state: Any, questions: Any, *, router: Any = None) -> dict:
+ """Routing decision only: no forward pass."""
+ state_d = validate_state(state)
+ questions_d = validate_questions(questions)
+ if router is None:
+ raise ToolError("models_not_ready", "Router is not loaded")
+ if not hasattr(router, "route"):
+ raise ToolError("internal_error", "router has no route() method")
+ return _decision_to_dict(router.route(state_d, questions_d))
+
+
def _resident_or_load(router: Any, name: str) -> Any:
"""The resident agent for checkpoint ``name``, loading on demand when possible.
@@ -252,21 +268,8 @@ def laya_shortlist(
raise ToolError("models_not_ready", "Router is not loaded (auto mode)")
if not hasattr(router, "route"):
raise ToolError("internal_error", "router has no route() method")
- decision = router.route(state_d, questions_d)
- if isinstance(decision, dict):
- routed = decision.get("model")
- routing = {
- "model": routed,
- "repo": decision.get("repo"),
- "reason": decision.get("reason"),
- }
- else:
- routed = getattr(decision, "model", None)
- routing = {
- "model": routed,
- "repo": getattr(decision, "repo", None),
- "reason": getattr(decision, "reason", None),
- }
+ routing = _decision_to_dict(router.route(state_d, questions_d))
+ routed = routing["model"]
if not isinstance(routed, str) or not routed:
raise ToolError("internal_error", "router.route returned no model")
predict_target = router
@@ -378,3 +381,165 @@ def laya_status(*, router: Any = None, loaded: list[str] | None = None, preload:
"router_ready": router is not None,
"package_versions": versions,
}
+
+
+# --- batch tools ----------------------------------------------------------
+
+def _validate_batch_model(value: Any, where: str) -> str | None:
+ """``model`` as a Router routing override: "auto" (or absent) means None so
+ Router.route resolves the checkpoint per request; a name pins it. Kept
+ separate from ``validate_model``, whose "auto" means "answer via
+ router.predict" on the single-request path.
+ """
+ if value is None or value == "auto":
+ return None
+ if value not in VALID_MODELS:
+ raise ToolError(
+ "invalid_model",
+ f"{where}['model'] must be one of {sorted(VALID_MODELS)}, got {value!r}",
+ )
+ return value
+
+
+def _validate_batch_str(value: Any, where: str, allow_empty: bool = False) -> str:
+ if not isinstance(value, str) or (not allow_empty and not value.strip()):
+ raise ToolError("invalid_request", f"{where} must be a non-empty string, got {value!r}")
+ return value
+
+
+def _validate_batch_item(request: Any, i: int) -> dict:
+ where = "requests[%d]" % i
+ if not isinstance(request, dict):
+ raise ToolError("invalid_request", "%s must be an object with 'state' and 'questions'" % where)
+ item: dict[str, Any] = {
+ "state": validate_state(request.get("state")),
+ "questions": validate_questions(request.get("questions")),
+ }
+ if "model" in request:
+ model = _validate_batch_model(request["model"], where)
+ if model is not None:
+ item["model"] = model
+ for key in ("task", "lang"):
+ if key in request:
+ item[key] = _validate_batch_str(request[key], "%s[%r]" % (where, key))
+ if "lang_guess" in request:
+ item["lang_guess"] = request["lang_guess"]
+ return item
+
+
+def validate_batch_requests(requests: Any) -> list[dict]:
+ """Validate a tool payload of many requests, preserving order.
+
+ Same per-item validation as ``laya_predict``/``laya_route`` (state, questions,
+ optional model/task/lang/lang_guess overrides), run before any model loads so
+ one malformed item fails the whole call instead of a partial batch.
+ """
+ if not isinstance(requests, list) or not requests:
+ raise ToolError(
+ "invalid_request",
+ "requests must be a non-empty array of {state, questions, model?, task?, lang?, lang_guess?} objects",
+ )
+ return [_validate_batch_item(request, i) for i, request in enumerate(requests)]
+
+
+def _validate_batch_size(batch_size: Any) -> int | None:
+ if batch_size is None:
+ return None
+ if isinstance(batch_size, bool) or not isinstance(batch_size, int) or batch_size < 1:
+ raise ToolError("invalid_batch_size", f"batch_size must be a positive integer, got {batch_size!r}")
+ return batch_size
+
+
+def laya_predict_batch(
+ requests: Any,
+ batch_size: Any = None,
+ *,
+ router: Any = None,
+) -> dict:
+ """Answer many requests in one call: ``Router.predict_batch`` over MCP.
+
+ ``laya_predict`` scores one request per round trip; an agent with many
+ tickets to triage pays a routing pass and a separate forward pass every
+ time. This tool hands the whole list to ``Router.predict_batch``, which
+ groups requests by routed checkpoint and shares forward passes between
+ requests with the same question schema, and returns the answers in input
+ order. Every item carries the same ``answers``/``routing``/``device``
+ fields as ``laya_predict``. Requests are validated up front, so one
+ malformed item errors before any model loads.
+ """
+ items = validate_batch_requests(requests)
+ size = _validate_batch_size(batch_size)
+ if router is None:
+ raise ToolError("models_not_ready", "Router is not loaded")
+ if not hasattr(router, "predict_batch"):
+ raise ToolError("internal_error", "router has no predict_batch() method")
+
+ started = time.perf_counter()
+ try:
+ results = router.predict_batch(items, batch_size=size) if size is not None \
+ else router.predict_batch(items)
+ except TypeError as exc:
+ # A Router without batch support raises at the call itself; anything
+ # else is a real bug and must surface unchanged.
+ raise ToolError(
+ "internal_error",
+ "router.predict_batch failed: %s: %s" % (type(exc).__name__, exc),
+ ) from exc
+ latency_ms = (time.perf_counter() - started) * 1000.0
+
+ if not isinstance(results, list) or len(results) != len(items):
+ raise ToolError(
+ "internal_error",
+ "predict_batch returned %r results for %d requests" % (
+ len(results) if isinstance(results, list) else type(results).__name__, len(items)),
+ )
+
+ out_requests: list[dict[str, Any]] = []
+ for result in results:
+ norm = _normalize_result(result)
+ entry: dict[str, Any] = {"answers": norm["answers"], "routing": norm["routing"] or {}}
+ device = _reading_device(router, None, "auto", norm["routing"])
+ if device:
+ entry["device"] = device
+ out_requests.append(entry)
+
+ model_counts: dict[str, int] = {}
+ for entry in out_requests:
+ name = entry["routing"].get("model")
+ key = name if isinstance(name, str) and name else "unknown"
+ model_counts[key] = model_counts.get(key, 0) + 1
+
+ return {
+ "requests": out_requests,
+ "model_counts": model_counts,
+ "total_latency_ms": round(latency_ms, 3),
+ "per_request_latency_ms": round(latency_ms / len(items), 3),
+ }
+
+
+def laya_route_batch(requests: Any, *, router: Any = None) -> dict:
+ """Routing decisions for many requests: no forward pass, no checkpoint loads.
+
+ The batch form of ``laya_route``, mirroring ``Router.route_batch``: it
+ reports which checkpoint each request *would* answer from so clients can
+ inspect or aggregate a workload's routing before paying any load cost.
+ """
+ items = validate_batch_requests(requests)
+ if router is None:
+ raise ToolError("models_not_ready", "Router is not loaded")
+ if not hasattr(router, "route_batch"):
+ raise ToolError("internal_error", "router has no route_batch() method")
+ decisions = router.route_batch(items)
+ if not isinstance(decisions, list) or len(decisions) != len(items):
+ raise ToolError(
+ "internal_error",
+ "route_batch returned %r decisions for %d requests" % (
+ len(decisions) if isinstance(decisions, list) else type(decisions).__name__, len(items)),
+ )
+ out = [_decision_to_dict(decision) for decision in decisions]
+ model_counts: dict[str, int] = {}
+ for entry in out:
+ name = entry["model"]
+ key = name if isinstance(name, str) and name else "unknown"
+ model_counts[key] = model_counts.get(key, 0) + 1
+ return {"decisions": out, "model_counts": model_counts}
diff --git a/tests/test_mcp.py b/tests/test_mcp.py
index 4d3afab5..d0a97512 100644
--- a/tests/test_mcp.py
+++ b/tests/test_mcp.py
@@ -30,10 +30,13 @@ from laya.mcp.server import _models_from_env, server as mcp_server # noqa: E402
from laya.mcp.tools import ( # noqa: E402
ToolError,
laya_predict,
+ laya_predict_batch,
laya_preset,
laya_route,
+ laya_route_batch,
laya_shortlist,
laya_status,
+ validate_batch_requests,
validate_model,
validate_preset,
validate_questions,
@@ -505,6 +508,168 @@ def test_shortlist():
and out["routing"]["reason"] == "explicit model", repr(out["routing"]))
+# --- batch tools (mocked router, no weights) ----------------------------------
+
+class BatchRouter(FakeRouter):
+ """Records each batch call; returns one result per request, input order."""
+
+ def __init__(self):
+ self.predict_batch_calls = []
+ self.route_batch_calls = []
+
+ def _answer_for(self, request):
+ answers = {}
+ for name, spec in request["questions"].items():
+ if spec["type"] == "choice":
+ answers[name] = {"choice": "billing", "confidence": 0.9}
+ elif spec["type"] == "score":
+ answers[name] = {"score": 1.5, "confidence": 0.8}
+ else:
+ answers[name] = {"noul": 0.7, "confidence": 0.9}
+ return {"answers": answers,
+ "routing": {"model": request.get("model") or "english",
+ "repo": "fake/laya", "reason": "batch route"}}
+
+ def predict_batch(self, requests, batch_size=None):
+ self.predict_batch_calls.append((list(requests), batch_size))
+ return [self._answer_for(request) for request in requests]
+
+ def route_batch(self, requests):
+ self.route_batch_calls.append(list(requests))
+ return [{"model": request.get("model") or "english", "repo": "fake/laya",
+ "reason": "batch route"} for request in requests]
+
+
+class ShortRouter:
+ """Router whose batches misreport their own size."""
+
+ def predict_batch(self, requests, batch_size=None):
+ return []
+
+ def route_batch(self, requests):
+ return []
+
+
+BATCH_REQUESTS = [
+ {"state": {"body": "refund please"}, "questions": QUESTIONS},
+ {"state": {"body": "please refund invoice 2"}, "questions": QUESTIONS,
+ "model": "english", "task": "massive", "lang": "en"},
+ {"state": {"body": "mera account double charge hua"},
+ "questions": {"triage": {"type": "noul", "instructions": "Leaving?"}},
+ "lang": "hi"},
+]
+
+
+def test_batch_validation():
+ for bad in (None, {}, "x", {"state": STATE, "questions": QUESTIONS}):
+ expect_tool_error("batch/requests_bad_%r" % (type(bad).__name__,),
+ lambda b=bad: validate_batch_requests(b), "invalid_request")
+ expect_tool_error("batch/requests_empty",
+ lambda: validate_batch_requests([]), "invalid_request")
+ expect_tool_error("batch/item_not_object",
+ lambda: validate_batch_requests(["x"]), "invalid_request")
+ expect_tool_error("batch/item_missing_state",
+ lambda: validate_batch_requests([{"questions": QUESTIONS}]), "invalid_state")
+ expect_tool_error("batch/item_missing_questions",
+ lambda: validate_batch_requests([{"state": STATE}]), "invalid_questions")
+ expect_tool_error("batch/item_bad_questions",
+ lambda: validate_batch_requests([{"state": STATE, "questions": {}}]),
+ "invalid_questions")
+ expect_tool_error("batch/item_bad_model",
+ lambda: validate_batch_requests(
+ [{"state": STATE, "questions": QUESTIONS, "model": "gpt4"}]),
+ "invalid_model")
+ for key, value in (("task", 3), ("lang", ""), ("lang", [])):
+ expect_tool_error("batch/item_bad_%s" % key,
+ lambda k=key, v=value: validate_batch_requests(
+ [{"state": STATE, "questions": QUESTIONS, k: v}]),
+ "invalid_request")
+
+ out = validate_batch_requests(BATCH_REQUESTS)
+ ok("batch/validation_preserves_order", [item["state"] for item in out]
+ == [request["state"] for request in BATCH_REQUESTS])
+ ok("batch/validation_keeps_overrides",
+ out[1].get("model") == "english" and out[1].get("task") == "massive"
+ and out[1].get("lang") == "en" and out[2].get("lang") == "hi", repr(out[1]))
+ # "auto" is the same thing as absent: Router.route resolves the checkpoint.
+ out = validate_batch_requests([{"state": STATE, "questions": QUESTIONS, "model": "auto"}])
+ ok("batch/validation_auto_dropped", "model" not in out[0], repr(out[0]))
+ # Keys the Router would never expect are dropped, not forwarded.
+ out = validate_batch_requests([{"state": STATE, "questions": QUESTIONS, "temperature": 0}])
+ ok("batch/validation_unknown_key_dropped", set(out[0]) == {"state", "questions"}, repr(out[0]))
+
+
+def test_batch_predict():
+ expect_tool_error("batch/predict_no_router",
+ lambda: laya_predict_batch(BATCH_REQUESTS, router=None), "models_not_ready")
+ expect_tool_error("batch/predict_no_method",
+ lambda: laya_predict_batch(BATCH_REQUESTS, router=FakeRouter()),
+ "internal_error")
+ for bad_size in (True, 0, -2, 2.5, "3"):
+ expect_tool_error("batch/predict_bad_size_%r" % (bad_size,),
+ lambda s=bad_size: laya_predict_batch(
+ BATCH_REQUESTS, batch_size=s, router=BatchRouter()),
+ "invalid_batch_size")
+ # Validation runs before the router is touched: one bad item, no partial batch.
+ router = BatchRouter()
+ expect_tool_error("batch/predict_validates_first",
+ lambda: laya_predict_batch([{"state": STATE}, {"state": STATE, "questions": QUESTIONS}],
+ router=router),
+ "invalid_questions")
+ ok("batch/predict_not_called_on_bad_input", router.predict_batch_calls == [])
+
+ router = BatchRouter()
+ out = laya_predict_batch(BATCH_REQUESTS, batch_size=8, router=router)
+ ok("batch/predict_one_call", len(router.predict_batch_calls) == 1,
+ repr(len(router.predict_batch_calls)))
+ forwarded, size = router.predict_batch_calls[0]
+ ok("batch/predict_size_forwarded", size == 8, repr(size))
+ ok("batch/predict_items_forwarded", [item["state"] for item in forwarded]
+ == [request["state"] for request in BATCH_REQUESTS])
+
+ ok("batch/predict_keys", set(out) == {"requests", "model_counts",
+ "total_latency_ms", "per_request_latency_ms"}, repr(sorted(out)))
+ ok("batch/predict_input_order", [entry["answers"].get("triage", {}).get("noul")
+ for entry in out["requests"]] == [None, None, 0.7],
+ repr(out["requests"]))
+ ok("batch/predict_department", out["requests"][0]["answers"]["department"]["choice"] == "billing")
+ ok("batch/predict_routing", out["requests"][1]["routing"]["model"] == "english")
+ ok("batch/predict_device_resident", out["requests"][0].get("device") == "cpu",
+ repr(out["requests"][0].get("device")))
+ ok("batch/predict_counts", out["model_counts"] == {"english": 3}, repr(out["model_counts"]))
+ ok("batch/predict_latency", out["total_latency_ms"] >= 0
+ and abs(out["per_request_latency_ms"] - out["total_latency_ms"] / 3) < 0.01,
+ repr(out["per_request_latency_ms"]))
+
+ # batch_size unset must not be forwarded as None (strict old stubs included).
+ router = BatchRouter()
+ laya_predict_batch(BATCH_REQUESTS, router=router)
+ ok("batch/predict_size_default", router.predict_batch_calls[0][1] is None)
+
+ expect_tool_error("batch/predict_count_mismatch",
+ lambda: laya_predict_batch(BATCH_REQUESTS, router=ShortRouter()),
+ "internal_error")
+
+
+def test_batch_route():
+ expect_tool_error("batch/route_no_router",
+ lambda: laya_route_batch(BATCH_REQUESTS, router=None), "models_not_ready")
+ expect_tool_error("batch/route_no_method",
+ lambda: laya_route_batch(BATCH_REQUESTS, router=FakeRouter()),
+ "internal_error")
+ router = BatchRouter()
+ out = laya_route_batch(BATCH_REQUESTS, router=router)
+ ok("batch/route_one_call", len(router.route_batch_calls) == 1)
+ ok("batch/route_decisions", len(out["decisions"]) == 3
+ and set(out["decisions"][0]) == {"model", "repo", "reason"}, repr(out["decisions"][0]))
+ ok("batch/route_counts", out["model_counts"] == {"english": 3}, repr(out["model_counts"]))
+ # Route-only never predicts.
+ ok("batch/route_no_forward", router.predict_batch_calls == [])
+ expect_tool_error("batch/route_count_mismatch",
+ lambda: laya_route_batch(BATCH_REQUESTS, router=ShortRouter()),
+ "internal_error")
+
+
def test_timeout_removed():
# The per-call timeout was removed: a ThreadPoolExecutor shutdown waits for
# the work anyway, and MCP clients apply their own request timeout. The tool
@@ -513,7 +678,8 @@ def test_timeout_removed():
import laya.mcp.tools as tools_mod
- for fn in (laya_predict, laya_route, laya_preset, laya_shortlist):
+ for fn in (laya_predict, laya_route, laya_preset, laya_shortlist,
+ laya_predict_batch, laya_route_batch):
ok("timeout/param_absent_%s" % fn.__name__, "timeout" not in inspect.signature(fn).parameters)
ok("timeout/executor_absent", "ThreadPoolExecutor" not in inspect.getsource(tools_mod))
@@ -543,17 +709,22 @@ def test_models_from_env():
def test_server_registration():
tools = asyncio.run(mcp_server.list_tools())
names = sorted(t.name for t in tools)
- ok("server/tool_names", names == ["laya_predict", "laya_preset", "laya_route", "laya_shortlist", "laya_status"], repr(names))
- decision = {"laya_predict", "laya_route", "laya_preset", "laya_shortlist"}
+ ok("server/tool_names", names == ["laya_predict", "laya_predict_batch", "laya_preset",
+ "laya_route", "laya_route_batch", "laya_shortlist",
+ "laya_status"], repr(names))
+ decision = {"laya_predict", "laya_route", "laya_preset", "laya_shortlist",
+ "laya_predict_batch", "laya_route_batch"}
for t in tools:
desc = (t.description or "").lower()
ok("server/desc_%s_nonempty" % t.name, bool(desc.strip()), repr(desc))
- # The guardrails constant is on the three decision tools only;
+ # The guardrails constant is on the decision tools only;
# laya_status reports instead of deciding.
if t.name in decision:
ok("server/desc_%s_guardrail" % t.name, "do not use" in desc)
if t.name == "laya_predict":
ok("server/desc_noul_labels", "optional labels" in desc)
+ if t.name.endswith("_batch"):
+ ok("server/desc_%s_requests" % t.name, "non-empty array" in desc)
test_device()
@@ -563,6 +734,9 @@ test_schema()
test_shape()
test_question_forwarding()
test_shortlist()
+test_batch_validation()
+test_batch_predict()
+test_batch_route()
test_timeout_removed()
test_models_from_env()
test_server_registration()
diff --git a/tests/test_mcp_local_e2e.py b/tests/test_mcp_local_e2e.py
index 6ccca90e..cb62643a 100644
--- a/tests/test_mcp_local_e2e.py
+++ b/tests/test_mcp_local_e2e.py
@@ -2,7 +2,8 @@
Launches ``python -m laya.mcp.server`` as a subprocess and speaks MCP over
stdin/stdout (newline-delimited JSON-RPC), exactly like a real MCP client.
-Exercises laya_predict, laya_preset, laya_route, laya_shortlist and laya_status against the
+Exercises laya_predict, laya_predict_batch, laya_route, laya_route_batch,
+laya_preset, laya_shortlist and laya_status against the
live checkpoints (downloaded via huggingface_hub on first run, cached
afterwards).
@@ -115,7 +116,9 @@ def main():
tools = client.request("tools/list", {})
names = sorted(t["name"] for t in tools.get("tools", []))
- ok("e2e/tool_names", names == ["laya_predict", "laya_preset", "laya_route", "laya_shortlist", "laya_status"], repr(names))
+ ok("e2e/tool_names", names == ["laya_predict", "laya_predict_batch", "laya_preset",
+ "laya_route", "laya_route_batch", "laya_shortlist",
+ "laya_status"], repr(names))
ticket = {
"state": {
@@ -159,6 +162,50 @@ def main():
repr(payload.get("device")))
ok("e2e/predict_latency", 0 < payload.get("latency_ms", -1) < 60_000, repr(payload.get("latency_ms")))
ok("e2e/predict_billing_wins", ans["choice"] == "billing", "ambiguous ticket -> expect billing")
+ solo_routing = payload["routing"]["model"]
+
+ # laya_predict_batch: the same English ticket plus a German one
+ # with an explicit `lang` override, answered in one call. The
+ # English answer must be decision-identical to the solo
+ # laya_predict above -- batching groups and shares forward passes,
+ # it must not change what the model decides.
+ batch_arguments = {
+ "requests": [
+ {"state": ticket["state"], "questions": ticket["questions"]},
+ {"state": {"body": "Mein Konto wurde zweimal belastet, bitte erstatten Sie das."},
+ "questions": ticket["questions"], "lang": "de"},
+ ],
+ }
+ result = client.request("tools/call",
+ {"name": "laya_predict_batch", "arguments": batch_arguments})
+ payload = json.loads(result["content"][0]["text"])
+ ok("e2e/batch_not_error", not result.get("isError"), repr(result.get("isError")))
+ entries = payload.get("requests") or []
+ # Entry 0 is the English ticket, so its routing must equal the solo
+ # laya_predict decision made for the same state above: one call per
+ # request, input order preserved.
+ ok("e2e/batch_input_order", len(entries) == 2, repr([e.get("routing") for e in entries]))
+ ok("e2e/batch_routes_like_solo",
+ (entries[0].get("routing") or {}).get("model") == solo_routing,
+ "solo=%r batch=%r" % (solo_routing, (entries[0].get("routing") or {}).get("model")))
+ ok("e2e/batch_decision_parity",
+ (entries[0].get("answers") or {}).get("department", {}).get("choice") == ans["choice"],
+ "solo=%r batch=%r" % (ans.get("choice"),
+ (entries[0].get("answers") or {}).get("department", {}).get("choice")))
+ ok("e2e/batch_counts", sum((payload.get("model_counts") or {}).values()) == 2,
+ repr(payload.get("model_counts")))
+ ok("e2e/batch_latency", 0 < payload.get("total_latency_ms", -1) < 60_000
+ and 0 < payload.get("per_request_latency_ms", -1) <= payload["total_latency_ms"],
+ repr(payload.get("total_latency_ms")))
+
+ result = client.request("tools/call",
+ {"name": "laya_route_batch", "arguments": batch_arguments})
+ payload = json.loads(result["content"][0]["text"])
+ ok("e2e/route_batch_not_error", not result.get("isError"), repr(result.get("isError")))
+ decisions = payload.get("decisions") or []
+ ok("e2e/route_batch_shape", len(decisions) == 2
+ and all(d.get("model") in ("english", "multilingual", "typed-decisions") and d.get("reason")
+ for d in decisions), repr(decisions))
result = client.request("tools/call", {"name": "laya_route", "arguments": ticket})
payload = json.loads(result["content"][0]["text"])