Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(mcp): add laya_predict_batch and laya_route_batch tools
Aashishd52056bc6491
diff --git a/laya/mcp/server.py b/laya/mcp/server.pyindex 0618af61..77debc0f 100644--- a/laya/mcp/server.py+++ b/laya/mcp/server.py@@ -40,8 +40,10 @@ from .device import env_devicefrom .tools import (ToolError,laya_predict,+ laya_predict_batch,laya_preset,laya_route,+ laya_route_batch,laya_shortlist,laya_status,)@@ -63,8 +65,8 @@ _ROUTER_LOCK = threading.Lock()# two, and auto-routing never selects it)._DEFAULT_MODELS = ("english", "multilingual")-# Shared usage guardrails for the three decision tools. laya_status reports-# instead of deciding, so it does not carry them.+# Shared usage guardrails for the decision tools. laya_status reports instead+# of deciding, so it does not carry them._GUARDRAILS = ("Use for structured decisions only: choice (finite labels), score (ordinal rubric), ""noul (calibrated P(true)). One forward pass ~33ms (GPU) / ~200ms (CPU). "@@ -199,6 +201,49 @@ def laya_predict_tool(state: dict, questions: dict, model: str = "auto") -> str:)+ name="laya_predict_batch",+ description=(+ "Answer many typed-question requests in one call: requests is a non-empty array of "+ "{state, questions, model?, task?, lang?, lang_guess?} objects, each with the same "+ "questions schema as laya_predict. Requests are routed first, grouped by checkpoint, "+ "and share forward passes when their question schemas match, so scoring many "+ "requests costs one round trip instead of N. Returns answers in input order with "+ "per-request routing and device, plus model_counts and batch latency. "+ + _GUARDRAILS+ ),+)+def laya_predict_batch_tool(requests: list, batch_size: int = 0) -> str:+ """Answer many typed-question requests in one batched call."""+ router = _router_or_error()+ # batch_size=0 means "unset": MCP clients send defaults eagerly, and+ # None is what Router.predict_batch takes as "no forward-pass cap".+ return _wrap(+ laya_predict_batch,+ requests=requests,+ batch_size=batch_size or None,+ router=router,+ )+++ name="laya_route_batch",+ description=(+ "Decide which Laya checkpoint would answer each request, without running any "+ "forward pass or loading a checkpoint: requests is a non-empty array of "+ "{state, questions, model?, task?, lang?, lang_guess?} objects. Use this to "+ "inspect or aggregate the routing of a workload before paying model-load cost. "+ "Returns one {model, repo, reason} decision per request in input order, plus "+ "model_counts. "+ + _GUARDRAILS+ ),+)+def laya_route_batch_tool(requests: list) -> str:+ """Route many requests to checkpoints without a forward pass."""+ router = _router_or_error()+ return _wrap(laya_route_batch, requests=requests, router=router)++@server.tool(name="laya_shortlist",description=(diff --git a/laya/mcp/tools.py b/laya/mcp/tools.pyindex 810f5edb..ca117b90 100644--- a/laya/mcp/tools.py+++ b/laya/mcp/tools.py@@ -117,6 +117,31 @@ def _normalize_answers(raw: Any) -> dict:return raw+def _normalize_result(result: Any) -> dict:+ if not isinstance(result, dict):+ raise ToolError("internal_error", "predict returned non-object")+ # Router.predict and Agent.system_one both return the system_one payload,+ # which always carries an "answers" object (empty for empty questions).+ try:+ answers = result["answers"]+ except KeyError:+ raise ToolError("internal_error", "predict result has no 'answers' object") from None+ return {"answers": _normalize_answers(answers), "routing": result.get("routing")}+++def _reading_device(router: Any, agent: Any, model_name: str, routing: dict | None) -> str | None:+ """Real device of the checkpoint that answered: Agent.device reflects a+ silent GPU -> CPU fallback. Omitted when it cannot be read, rather than+ guessed.+ """+ if model_name != "auto" and agent is not None:+ return agent_device(agent)+ model_used = (routing or {}).get("model")+ if isinstance(model_used, str) and model_used:+ return agent_device(router_agent(router, model_used))+ return None++def laya_predict(state: Any,questions: Any,@@ -148,22 +173,10 @@ def laya_predict(result = _run()latency_ms = (time.perf_counter() - started) * 1000.0- if not isinstance(result, dict):- raise ToolError("internal_error", "predict returned non-object")- # Router.predict and Agent.system_one both return the system_one payload,- # which always carries an "answers" object (empty for empty questions).- answers = _normalize_answers(result["answers"])- routing = result.get("routing") or {"model": model_name, "repo": None, "reason": "explicit model"}- # Real device of the checkpoint that answered: Agent.device reflects a- # silent GPU -> CPU fallback. Omitted when it cannot be read, rather than- # guessed.- device = None- if model_name != "auto" and agent is not None:- device = agent_device(agent)- else:- model_used = routing.get("model")- if isinstance(model_used, str) and model_used:- device = agent_device(router_agent(router, model_used))+ norm = _normalize_result(result)+ answers = norm["answers"]+ routing = norm["routing"] or {"model": model_name, "repo": None, "reason": "explicit model"}+ device = _reading_device(router, agent, model_name, norm["routing"])out: dict[str, Any] = {"answers": answers,"routing": routing,@@ -174,15 +187,7 @@ def laya_predict(return out-def laya_route(state: Any, questions: Any, *, router: Any = None) -> dict:- """Routing decision only: no forward pass."""- state_d = validate_state(state)- questions_d = validate_questions(questions)- if router is None:- raise ToolError("models_not_ready", "Router is not loaded")- if not hasattr(router, "route"):- raise ToolError("internal_error", "router has no route() method")- decision = router.route(state_d, questions_d)+def _decision_to_dict(decision: Any) -> dict:if isinstance(decision, dict):return {"model": decision.get("model"),@@ -196,6 +201,17 @@ def laya_route(state: Any, questions: Any, *, router: Any = None) -> dict:}+def laya_route(state: Any, questions: Any, *, router: Any = None) -> dict:+ """Routing decision only: no forward pass."""+ state_d = validate_state(state)+ questions_d = validate_questions(questions)+ if router is None:+ raise ToolError("models_not_ready", "Router is not loaded")+ if not hasattr(router, "route"):+ raise ToolError("internal_error", "router has no route() method")+ return _decision_to_dict(router.route(state_d, questions_d))++def _resident_or_load(router: Any, name: str) -> Any:"""The resident agent for checkpoint ``name``, loading on demand when possible.@@ -252,21 +268,8 @@ def laya_shortlist(raise ToolError("models_not_ready", "Router is not loaded (auto mode)")if not hasattr(router, "route"):raise ToolError("internal_error", "router has no route() method")- decision = router.route(state_d, questions_d)- if isinstance(decision, dict):- routed = decision.get("model")- routing = {- "model": routed,- "repo": decision.get("repo"),- "reason": decision.get("reason"),- }- else:- routed = getattr(decision, "model", None)- routing = {- "model": routed,- "repo": getattr(decision, "repo", None),- "reason": getattr(decision, "reason", None),- }+ routing = _decision_to_dict(router.route(state_d, questions_d))+ routed = routing["model"]if not isinstance(routed, str) or not routed:raise ToolError("internal_error", "router.route returned no model")predict_target = router@@ -378,3 +381,165 @@ def laya_status(*, router: Any = None, loaded: list[str] | None = None, preload:"router_ready": router is not None,"package_versions": versions,}+++# --- batch tools ----------------------------------------------------------++def _validate_batch_model(value: Any, where: str) -> str | None:+ """``model`` as a Router routing override: "auto" (or absent) means None so+ Router.route resolves the checkpoint per request; a name pins it. Kept+ separate from ``validate_model``, whose "auto" means "answer via+ router.predict" on the single-request path.+ """+ if value is None or value == "auto":+ return None+ if value not in VALID_MODELS:+ raise ToolError(+ "invalid_model",+ f"{where}['model'] must be one of {sorted(VALID_MODELS)}, got {value!r}",+ )+ return value+++def _validate_batch_str(value: Any, where: str, allow_empty: bool = False) -> str:+ if not isinstance(value, str) or (not allow_empty and not value.strip()):+ raise ToolError("invalid_request", f"{where} must be a non-empty string, got {value!r}")+ return value+++def _validate_batch_item(request: Any, i: int) -> dict:+ where = "requests[%d]" % i+ if not isinstance(request, dict):+ raise ToolError("invalid_request", "%s must be an object with 'state' and 'questions'" % where)+ item: dict[str, Any] = {+ "state": validate_state(request.get("state")),+ "questions": validate_questions(request.get("questions")),+ }+ if "model" in request:+ model = _validate_batch_model(request["model"], where)+ if model is not None:+ item["model"] = model+ for key in ("task", "lang"):+ if key in request:+ item[key] = _validate_batch_str(request[key], "%s[%r]" % (where, key))+ if "lang_guess" in request:+ item["lang_guess"] = request["lang_guess"]+ return item+++def validate_batch_requests(requests: Any) -> list[dict]:+ """Validate a tool payload of many requests, preserving order.++ Same per-item validation as ``laya_predict``/``laya_route`` (state, questions,+ optional model/task/lang/lang_guess overrides), run before any model loads so+ one malformed item fails the whole call instead of a partial batch.+ """+ if not isinstance(requests, list) or not requests:+ raise ToolError(+ "invalid_request",+ "requests must be a non-empty array of {state, questions, model?, task?, lang?, lang_guess?} objects",+ )+ return [_validate_batch_item(request, i) for i, request in enumerate(requests)]+++def _validate_batch_size(batch_size: Any) -> int | None:+ if batch_size is None:+ return None+ if isinstance(batch_size, bool) or not isinstance(batch_size, int) or batch_size < 1:+ raise ToolError("invalid_batch_size", f"batch_size must be a positive integer, got {batch_size!r}")+ return batch_size+++def laya_predict_batch(+ requests: Any,+ batch_size: Any = None,+ *,+ router: Any = None,+) -> dict:+ """Answer many requests in one call: ``Router.predict_batch`` over MCP.++ ``laya_predict`` scores one request per round trip; an agent with many+ tickets to triage pays a routing pass and a separate forward pass every+ time. This tool hands the whole list to ``Router.predict_batch``, which+ groups requests by routed checkpoint and shares forward passes between+ requests with the same question schema, and returns the answers in input+ order. Every item carries the same ``answers``/``routing``/``device``+ fields as ``laya_predict``. Requests are validated up front, so one+ malformed item errors before any model loads.+ """+ items = validate_batch_requests(requests)+ size = _validate_batch_size(batch_size)+ if router is None:+ raise ToolError("models_not_ready", "Router is not loaded")+ if not hasattr(router, "predict_batch"):+ raise ToolError("internal_error", "router has no predict_batch() method")++ started = time.perf_counter()+ try:+ results = router.predict_batch(items, batch_size=size) if size is not None \+ else router.predict_batch(items)+ except TypeError as exc:+ # A Router without batch support raises at the call itself; anything+ # else is a real bug and must surface unchanged.+ raise ToolError(+ "internal_error",+ "router.predict_batch failed: %s: %s" % (type(exc).__name__, exc),+ ) from exc+ latency_ms = (time.perf_counter() - started) * 1000.0++ if not isinstance(results, list) or len(results) != len(items):+ raise ToolError(+ "internal_error",+ "predict_batch returned %r results for %d requests" % (+ len(results) if isinstance(results, list) else type(results).__name__, len(items)),+ )++ out_requests: list[dict[str, Any]] = []+ for result in results:+ norm = _normalize_result(result)+ entry: dict[str, Any] = {"answers": norm["answers"], "routing": norm["routing"] or {}}+ device = _reading_device(router, None, "auto", norm["routing"])+ if device:+ entry["device"] = device+ out_requests.append(entry)++ model_counts: dict[str, int] = {}+ for entry in out_requests:+ name = entry["routing"].get("model")+ key = name if isinstance(name, str) and name else "unknown"+ model_counts[key] = model_counts.get(key, 0) + 1++ return {+ "requests": out_requests,+ "model_counts": model_counts,+ "total_latency_ms": round(latency_ms, 3),+ "per_request_latency_ms": round(latency_ms / len(items), 3),+ }+++def laya_route_batch(requests: Any, *, router: Any = None) -> dict:+ """Routing decisions for many requests: no forward pass, no checkpoint loads.++ The batch form of ``laya_route``, mirroring ``Router.route_batch``: it+ reports which checkpoint each request *would* answer from so clients can+ inspect or aggregate a workload's routing before paying any load cost.+ """+ items = validate_batch_requests(requests)+ if router is None:+ raise ToolError("models_not_ready", "Router is not loaded")+ if not hasattr(router, "route_batch"):+ raise ToolError("internal_error", "router has no route_batch() method")+ decisions = router.route_batch(items)+ if not isinstance(decisions, list) or len(decisions) != len(items):+ raise ToolError(+ "internal_error",+ "route_batch returned %r decisions for %d requests" % (+ len(decisions) if isinstance(decisions, list) else type(decisions).__name__, len(items)),+ )+ out = [_decision_to_dict(decision) for decision in decisions]+ model_counts: dict[str, int] = {}+ for entry in out:+ name = entry["model"]+ key = name if isinstance(name, str) and name else "unknown"+ model_counts[key] = model_counts.get(key, 0) + 1+ return {"decisions": out, "model_counts": model_counts}diff --git a/tests/test_mcp.py b/tests/test_mcp.pyindex 4d3afab5..d0a97512 100644--- a/tests/test_mcp.py+++ b/tests/test_mcp.py@@ -30,10 +30,13 @@ from laya.mcp.server import _models_from_env, server as mcp_server # noqa: E402from laya.mcp.tools import ( # noqa: E402ToolError,laya_predict,+ laya_predict_batch,laya_preset,laya_route,+ laya_route_batch,laya_shortlist,laya_status,+ validate_batch_requests,validate_model,validate_preset,validate_questions,@@ -505,6 +508,168 @@ def test_shortlist():and out["routing"]["reason"] == "explicit model", repr(out["routing"]))+# --- batch tools (mocked router, no weights) ----------------------------------++class BatchRouter(FakeRouter):+ """Records each batch call; returns one result per request, input order."""++ def __init__(self):+ self.predict_batch_calls = []+ self.route_batch_calls = []++ def _answer_for(self, request):+ answers = {}+ for name, spec in request["questions"].items():+ if spec["type"] == "choice":+ answers[name] = {"choice": "billing", "confidence": 0.9}+ elif spec["type"] == "score":+ answers[name] = {"score": 1.5, "confidence": 0.8}+ else:+ answers[name] = {"noul": 0.7, "confidence": 0.9}+ return {"answers": answers,+ "routing": {"model": request.get("model") or "english",+ "repo": "fake/laya", "reason": "batch route"}}++ def predict_batch(self, requests, batch_size=None):+ self.predict_batch_calls.append((list(requests), batch_size))+ return [self._answer_for(request) for request in requests]++ def route_batch(self, requests):+ self.route_batch_calls.append(list(requests))+ return [{"model": request.get("model") or "english", "repo": "fake/laya",+ "reason": "batch route"} for request in requests]+++class ShortRouter:+ """Router whose batches misreport their own size."""++ def predict_batch(self, requests, batch_size=None):+ return []++ def route_batch(self, requests):+ return []+++BATCH_REQUESTS = [+ {"state": {"body": "refund please"}, "questions": QUESTIONS},+ {"state": {"body": "please refund invoice 2"}, "questions": QUESTIONS,+ "model": "english", "task": "massive", "lang": "en"},+ {"state": {"body": "mera account double charge hua"},+ "questions": {"triage": {"type": "noul", "instructions": "Leaving?"}},+ "lang": "hi"},+]+++def test_batch_validation():+ for bad in (None, {}, "x", {"state": STATE, "questions": QUESTIONS}):+ expect_tool_error("batch/requests_bad_%r" % (type(bad).__name__,),+ lambda b=bad: validate_batch_requests(b), "invalid_request")+ expect_tool_error("batch/requests_empty",+ lambda: validate_batch_requests([]), "invalid_request")+ expect_tool_error("batch/item_not_object",+ lambda: validate_batch_requests(["x"]), "invalid_request")+ expect_tool_error("batch/item_missing_state",+ lambda: validate_batch_requests([{"questions": QUESTIONS}]), "invalid_state")+ expect_tool_error("batch/item_missing_questions",+ lambda: validate_batch_requests([{"state": STATE}]), "invalid_questions")+ expect_tool_error("batch/item_bad_questions",+ lambda: validate_batch_requests([{"state": STATE, "questions": {}}]),+ "invalid_questions")+ expect_tool_error("batch/item_bad_model",+ lambda: validate_batch_requests(+ [{"state": STATE, "questions": QUESTIONS, "model": "gpt4"}]),+ "invalid_model")+ for key, value in (("task", 3), ("lang", ""), ("lang", [])):+ expect_tool_error("batch/item_bad_%s" % key,+ lambda k=key, v=value: validate_batch_requests(+ [{"state": STATE, "questions": QUESTIONS, k: v}]),+ "invalid_request")++ out = validate_batch_requests(BATCH_REQUESTS)+ ok("batch/validation_preserves_order", [item["state"] for item in out]+ == [request["state"] for request in BATCH_REQUESTS])+ ok("batch/validation_keeps_overrides",+ out[1].get("model") == "english" and out[1].get("task") == "massive"+ and out[1].get("lang") == "en" and out[2].get("lang") == "hi", repr(out[1]))+ # "auto" is the same thing as absent: Router.route resolves the checkpoint.+ out = validate_batch_requests([{"state": STATE, "questions": QUESTIONS, "model": "auto"}])+ ok("batch/validation_auto_dropped", "model" not in out[0], repr(out[0]))+ # Keys the Router would never expect are dropped, not forwarded.+ out = validate_batch_requests([{"state": STATE, "questions": QUESTIONS, "temperature": 0}])+ ok("batch/validation_unknown_key_dropped", set(out[0]) == {"state", "questions"}, repr(out[0]))+++def test_batch_predict():+ expect_tool_error("batch/predict_no_router",+ lambda: laya_predict_batch(BATCH_REQUESTS, router=None), "models_not_ready")+ expect_tool_error("batch/predict_no_method",+ lambda: laya_predict_batch(BATCH_REQUESTS, router=FakeRouter()),+ "internal_error")+ for bad_size in (True, 0, -2, 2.5, "3"):+ expect_tool_error("batch/predict_bad_size_%r" % (bad_size,),+ lambda s=bad_size: laya_predict_batch(+ BATCH_REQUESTS, batch_size=s, router=BatchRouter()),+ "invalid_batch_size")+ # Validation runs before the router is touched: one bad item, no partial batch.+ router = BatchRouter()+ expect_tool_error("batch/predict_validates_first",+ lambda: laya_predict_batch([{"state": STATE}, {"state": STATE, "questions": QUESTIONS}],+ router=router),+ "invalid_questions")+ ok("batch/predict_not_called_on_bad_input", router.predict_batch_calls == [])++ router = BatchRouter()+ out = laya_predict_batch(BATCH_REQUESTS, batch_size=8, router=router)+ ok("batch/predict_one_call", len(router.predict_batch_calls) == 1,+ repr(len(router.predict_batch_calls)))+ forwarded, size = router.predict_batch_calls[0]+ ok("batch/predict_size_forwarded", size == 8, repr(size))+ ok("batch/predict_items_forwarded", [item["state"] for item in forwarded]+ == [request["state"] for request in BATCH_REQUESTS])++ ok("batch/predict_keys", set(out) == {"requests", "model_counts",+ "total_latency_ms", "per_request_latency_ms"}, repr(sorted(out)))+ ok("batch/predict_input_order", [entry["answers"].get("triage", {}).get("noul")+ for entry in out["requests"]] == [None, None, 0.7],+ repr(out["requests"]))+ ok("batch/predict_department", out["requests"][0]["answers"]["department"]["choice"] == "billing")+ ok("batch/predict_routing", out["requests"][1]["routing"]["model"] == "english")+ ok("batch/predict_device_resident", out["requests"][0].get("device") == "cpu",+ repr(out["requests"][0].get("device")))+ ok("batch/predict_counts", out["model_counts"] == {"english": 3}, repr(out["model_counts"]))+ ok("batch/predict_latency", out["total_latency_ms"] >= 0+ and abs(out["per_request_latency_ms"] - out["total_latency_ms"] / 3) < 0.01,+ repr(out["per_request_latency_ms"]))++ # batch_size unset must not be forwarded as None (strict old stubs included).+ router = BatchRouter()+ laya_predict_batch(BATCH_REQUESTS, router=router)+ ok("batch/predict_size_default", router.predict_batch_calls[0][1] is None)++ expect_tool_error("batch/predict_count_mismatch",+ lambda: laya_predict_batch(BATCH_REQUESTS, router=ShortRouter()),+ "internal_error")+++def test_batch_route():+ expect_tool_error("batch/route_no_router",+ lambda: laya_route_batch(BATCH_REQUESTS, router=None), "models_not_ready")+ expect_tool_error("batch/route_no_method",+ lambda: laya_route_batch(BATCH_REQUESTS, router=FakeRouter()),+ "internal_error")+ router = BatchRouter()+ out = laya_route_batch(BATCH_REQUESTS, router=router)+ ok("batch/route_one_call", len(router.route_batch_calls) == 1)+ ok("batch/route_decisions", len(out["decisions"]) == 3+ and set(out["decisions"][0]) == {"model", "repo", "reason"}, repr(out["decisions"][0]))+ ok("batch/route_counts", out["model_counts"] == {"english": 3}, repr(out["model_counts"]))+ # Route-only never predicts.+ ok("batch/route_no_forward", router.predict_batch_calls == [])+ expect_tool_error("batch/route_count_mismatch",+ lambda: laya_route_batch(BATCH_REQUESTS, router=ShortRouter()),+ "internal_error")++def test_timeout_removed():# The per-call timeout was removed: a ThreadPoolExecutor shutdown waits for# the work anyway, and MCP clients apply their own request timeout. The tool@@ -513,7 +678,8 @@ def test_timeout_removed():import laya.mcp.tools as tools_mod- for fn in (laya_predict, laya_route, laya_preset, laya_shortlist):+ for fn in (laya_predict, laya_route, laya_preset, laya_shortlist,+ laya_predict_batch, laya_route_batch):ok("timeout/param_absent_%s" % fn.__name__, "timeout" not in inspect.signature(fn).parameters)ok("timeout/executor_absent", "ThreadPoolExecutor" not in inspect.getsource(tools_mod))@@ -543,17 +709,22 @@ def test_models_from_env():def test_server_registration():tools = asyncio.run(mcp_server.list_tools())names = sorted(t.name for t in tools)- ok("server/tool_names", names == ["laya_predict", "laya_preset", "laya_route", "laya_shortlist", "laya_status"], repr(names))- decision = {"laya_predict", "laya_route", "laya_preset", "laya_shortlist"}+ ok("server/tool_names", names == ["laya_predict", "laya_predict_batch", "laya_preset",+ "laya_route", "laya_route_batch", "laya_shortlist",+ "laya_status"], repr(names))+ decision = {"laya_predict", "laya_route", "laya_preset", "laya_shortlist",+ "laya_predict_batch", "laya_route_batch"}for t in tools:desc = (t.description or "").lower()ok("server/desc_%s_nonempty" % t.name, bool(desc.strip()), repr(desc))- # The guardrails constant is on the three decision tools only;+ # The guardrails constant is on the decision tools only;# laya_status reports instead of deciding.if t.name in decision:ok("server/desc_%s_guardrail" % t.name, "do not use" in desc)if t.name == "laya_predict":ok("server/desc_noul_labels", "optional labels" in desc)+ if t.name.endswith("_batch"):+ ok("server/desc_%s_requests" % t.name, "non-empty array" in desc)test_device()@@ -563,6 +734,9 @@ test_schema()test_shape()test_question_forwarding()test_shortlist()+test_batch_validation()+test_batch_predict()+test_batch_route()test_timeout_removed()test_models_from_env()test_server_registration()diff --git a/tests/test_mcp_local_e2e.py b/tests/test_mcp_local_e2e.pyindex 6ccca90e..cb62643a 100644--- a/tests/test_mcp_local_e2e.py+++ b/tests/test_mcp_local_e2e.py@@ -2,7 +2,8 @@Launches ``python -m laya.mcp.server`` as a subprocess and speaks MCP overstdin/stdout (newline-delimited JSON-RPC), exactly like a real MCP client.-Exercises laya_predict, laya_preset, laya_route, laya_shortlist and laya_status against the+Exercises laya_predict, laya_predict_batch, laya_route, laya_route_batch,+laya_preset, laya_shortlist and laya_status against thelive checkpoints (downloaded via huggingface_hub on first run, cachedafterwards).@@ -115,7 +116,9 @@ def main():tools = client.request("tools/list", {})names = sorted(t["name"] for t in tools.get("tools", []))- ok("e2e/tool_names", names == ["laya_predict", "laya_preset", "laya_route", "laya_shortlist", "laya_status"], repr(names))+ ok("e2e/tool_names", names == ["laya_predict", "laya_predict_batch", "laya_preset",+ "laya_route", "laya_route_batch", "laya_shortlist",+ "laya_status"], repr(names))ticket = {"state": {@@ -159,6 +162,50 @@ def main():repr(payload.get("device")))ok("e2e/predict_latency", 0 < payload.get("latency_ms", -1) < 60_000, repr(payload.get("latency_ms")))ok("e2e/predict_billing_wins", ans["choice"] == "billing", "ambiguous ticket -> expect billing")+ solo_routing = payload["routing"]["model"]++ # laya_predict_batch: the same English ticket plus a German one+ # with an explicit `lang` override, answered in one call. The+ # English answer must be decision-identical to the solo+ # laya_predict above -- batching groups and shares forward passes,+ # it must not change what the model decides.+ batch_arguments = {+ "requests": [+ {"state": ticket["state"], "questions": ticket["questions"]},+ {"state": {"body": "Mein Konto wurde zweimal belastet, bitte erstatten Sie das."},+ "questions": ticket["questions"], "lang": "de"},+ ],+ }+ result = client.request("tools/call",+ {"name": "laya_predict_batch", "arguments": batch_arguments})+ payload = json.loads(result["content"][0]["text"])+ ok("e2e/batch_not_error", not result.get("isError"), repr(result.get("isError")))+ entries = payload.get("requests") or []+ # Entry 0 is the English ticket, so its routing must equal the solo+ # laya_predict decision made for the same state above: one call per+ # request, input order preserved.+ ok("e2e/batch_input_order", len(entries) == 2, repr([e.get("routing") for e in entries]))+ ok("e2e/batch_routes_like_solo",+ (entries[0].get("routing") or {}).get("model") == solo_routing,+ "solo=%r batch=%r" % (solo_routing, (entries[0].get("routing") or {}).get("model")))+ ok("e2e/batch_decision_parity",+ (entries[0].get("answers") or {}).get("department", {}).get("choice") == ans["choice"],+ "solo=%r batch=%r" % (ans.get("choice"),+ (entries[0].get("answers") or {}).get("department", {}).get("choice")))+ ok("e2e/batch_counts", sum((payload.get("model_counts") or {}).values()) == 2,+ repr(payload.get("model_counts")))+ ok("e2e/batch_latency", 0 < payload.get("total_latency_ms", -1) < 60_000+ and 0 < payload.get("per_request_latency_ms", -1) <= payload["total_latency_ms"],+ repr(payload.get("total_latency_ms")))++ result = client.request("tools/call",+ {"name": "laya_route_batch", "arguments": batch_arguments})+ payload = json.loads(result["content"][0]["text"])+ ok("e2e/route_batch_not_error", not result.get("isError"), repr(result.get("isError")))+ decisions = payload.get("decisions") or []+ ok("e2e/route_batch_shape", len(decisions) == 2+ and all(d.get("model") in ("english", "multilingual", "typed-decisions") and d.get("reason")+ for d in decisions), repr(decisions))result = client.request("tools/call", {"name": "laya_route", "arguments": ticket})payload = json.loads(result["content"][0]["text"])