Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
mcp: add laya_shortlist tool for many-option choice questions
Aashish5df6910eb0f0
diff --git a/README.md b/README.mdindex 507e230e..db591525 100644--- a/README.md+++ b/README.md@@ -719,7 +719,7 @@ override on your own data rather than treating `A`/`B` as a universal fix.Laya can be exposed as an [MCP](https://modelcontextprotocol.io) stdio server, so any MCPclient (OpenClaw, Claude Desktop, Cursor, ...) can call typed decisions as tools-(`laya_predict`, `laya_route`, `laya_preset`, `laya_status`) without writing glue code.+(`laya_predict`, `laya_route`, `laya_shortlist`, `laya_preset`, `laya_status`) without writing glue code.This is an **optional extra**: the core package has no `mcp` dependency.```bash@@ -752,7 +752,13 @@ package:| `LAYA_THREADS` | (torch default) | Same as `laya.serve`: cap torch intra-op threads for CPU inference; keep it at or below the physical core count |The tools return structured JSON (answers with probabilities, routing metadata, device,-`latency_ms`). As with the SDK, use it for structured decisions only; not for open Q&A or+`latency_ms`). `laya_shortlist` is the MCP form of [`predict_shortlist`](#honest-limits):+it shortlists a many-option choice question to its `k` most likely labels by embedding+similarity (mean-pooled from the answering checkpoint's own encoder, so no extra model is+downloaded), answers in one forward pass, and returns per-question shortlist metadata+(kept labels, cosine scores, `k`, option count). The guardrails shown on every decision+tool point clients here for >20-option choices. As with the SDK, use it for structured+decisions only; not for open Q&A ortext generation. Tests: `tests/test_mcp.py` (CI, no weights) and`tests/test_mcp_local_e2e.py` (local, real weights and a real stdio handshake).diff --git a/laya/mcp/server.py b/laya/mcp/server.pyindex eea0847a..d76e67f7 100644--- a/laya/mcp/server.py+++ b/laya/mcp/server.py@@ -42,6 +42,7 @@ from .tools import (laya_predict,laya_preset,laya_route,+ laya_shortlist,laya_status,)@@ -69,7 +70,7 @@ _GUARDRAILS = ("noul (calibrated P(true)). One forward pass ~33ms (GPU) / ~200ms (CPU). ""No text generation, so no hallucination. Do NOT use for open Q&A, summarization, ""rewriting, code, or multi-hop reasoning. Do NOT use for >20-option choice "- "questions without shortlisting."+ "questions without shortlisting (use the laya_shortlist tool).")@@ -197,6 +198,34 @@ def laya_predict_tool(state: dict, questions: dict, model: str = "auto") -> str:)+ name="laya_shortlist",+ description=(+ "Shortlist a many-option choice question to its k most likely labels by embedding "+ "similarity (mean-pooled from the answering checkpoint's own encoder, so no extra "+ "model is downloaded), then answer in one forward pass. Use this instead of "+ "laya_predict whenever a choice question has more options than the guardrails allow. "+ "Returns the answers plus per-question shortlist metadata (kept labels, cosine "+ "scores, k, option count). "+ + _GUARDRAILS+ ),+)+def laya_shortlist_tool(state: dict, questions: dict, model: str = "auto", k: int = 20) -> str:+ """Shortlist many-option choice questions, then answer."""+ # k's default mirrors laya.shortlist.DEFAULT_SHORTLIST_K; it is a literal+ # here so the MCP schema carries the default without importing numpy at+ # server start.+ router = _router_or_error()+ return _wrap(+ laya_shortlist,+ state=state,+ questions=questions,+ model=model,+ k=k,+ router=router,+ )++@server.tool(name="laya_preset",description=(diff --git a/laya/mcp/tools.py b/laya/mcp/tools.pyindex b3fc6b74..0e05be8a 100644--- a/laya/mcp/tools.py+++ b/laya/mcp/tools.py@@ -3,7 +3,7 @@from __future__ import annotationsimport time-from typing import Any, Callable, Protocol+from typing import Any, Callable, Protocol, Sequencefrom .device import agent_device, device_report, router_agent@@ -194,6 +194,124 @@ def laya_route(state: Any, questions: Any, *, router: Any = None) -> dict:}+def _resident_or_load(router: Any, name: str) -> Any:+ """The resident agent for checkpoint ``name``, loading on demand when possible.++ Read-only lookup first: ``router_agent`` never calls ``load()``. A miss+ falls back to ``Router.load``, the same on-demand build ``Router.predict``+ performs after routing, so a lazily preloaded server (LAYA_PRELOAD=0, or a+ checkpoint outside LAYA_MODELS) behaves exactly like ``laya_predict``.+ """+ resident = router_agent(router, name)+ if resident is not None:+ return resident+ load = getattr(router, "load", None)+ if load is None:+ raise ToolError("models_not_ready", f"checkpoint {name!r} is not loaded")+ return load(name)+++def laya_shortlist(+ state: Any,+ questions: Any,+ model: Any = "auto",+ k: Any = None,+ *,+ router: Any = None,+ agent: Any = None,+ embed_fn: Callable[[Sequence[str]], Any] | None = None,+) -> dict:+ """Shortlist many-option choice questions to ``k`` labels, then one predict.++ The shared guardrails tell clients not to run >20-option choice questions+ without shortlisting; this tool is that shortlisting (the in-process+ ``laya.shortlist.predict_shortlist`` pattern over MCP). Embeddings come+ from the answering checkpoint's own encoder (``embed_fn_from_agent``), so+ no extra model is downloaded; ``embed_fn`` is injectable for tests or for+ a dedicated bi-encoder.++ ``routing`` reports the real route decision in auto mode (the forward+ pass then runs with an explicit ``model=``, so routing happens once).+ """+ # Lazy: keeps numpy/shortlist out of module import for laya.mcp.tools.+ from laya.shortlist import DEFAULT_SHORTLIST_K, embed_fn_from_agent, predict_shortlist++ state_d = validate_state(state)+ questions_d = validate_questions(questions)+ model_name = validate_model(model)+ if k is None:+ k = DEFAULT_SHORTLIST_K+ if isinstance(k, bool) or not isinstance(k, int) or k < 1:+ raise ToolError("invalid_k", f"k must be a positive integer, got {k!r}")++ routing: dict[str, Any]+ if model_name == "auto":+ if router is None:+ raise ToolError("models_not_ready", "Router is not loaded (auto mode)")+ if not hasattr(router, "route"):+ raise ToolError("internal_error", "router has no route() method")+ decision = router.route(state_d, questions_d)+ if isinstance(decision, dict):+ routed = decision.get("model")+ routing = {+ "model": routed,+ "repo": decision.get("repo"),+ "reason": decision.get("reason"),+ }+ else:+ routed = getattr(decision, "model", None)+ routing = {+ "model": routed,+ "repo": getattr(decision, "repo", None),+ "reason": getattr(decision, "reason", None),+ }+ if not isinstance(routed, str) or not routed:+ raise ToolError("internal_error", "router.route returned no model")+ predict_target = router+ predict_kwargs: dict[str, Any] = {"model": routed}+ embed_agent = _resident_or_load(router, routed)+ elif agent is not None:+ routing = {"model": model_name, "repo": None, "reason": "explicit model"}+ predict_target = agent+ predict_kwargs = {}+ embed_agent = agent+ else:+ if router is None:+ raise ToolError("models_not_ready", "no agent/router loaded")+ routing = {"model": model_name, "repo": None, "reason": "explicit model"}+ predict_target = router+ predict_kwargs = {"model": model_name}+ embed_agent = _resident_or_load(router, model_name)++ if embed_fn is None:+ try:+ embed_fn = embed_fn_from_agent(embed_agent)+ except (AttributeError, TypeError, ValueError) as exc:+ raise ToolError(+ "models_not_ready",+ f"cannot build shortlist embeddings from checkpoint {routing['model']!r}: {exc}",+ ) from exc++ started = time.perf_counter()+ result = predict_shortlist(predict_target, state_d, questions_d, embed_fn, k=k, **predict_kwargs)+ latency_ms = (time.perf_counter() - started) * 1000.0++ if not isinstance(result, dict):+ raise ToolError("internal_error", "predict returned non-object")+ answers = _normalize_answers(result["answers"])+ # The answering checkpoint is the embedding checkpoint in every branch.+ device = agent_device(embed_agent)+ out: dict[str, Any] = {+ "answers": answers,+ "routing": routing,+ "shortlist": result.get("shortlist") or {},+ "latency_ms": round(latency_ms, 3),+ }+ if device:+ out["device"] = device+ return out++def laya_preset(preset: Any,state: Any,diff --git a/tests/test_mcp.py b/tests/test_mcp.pyindex b290fec6..da6ebdee 100644--- a/tests/test_mcp.py+++ b/tests/test_mcp.py@@ -32,6 +32,7 @@ from laya.mcp.tools import ( # noqa: E402laya_predict,laya_preset,laya_route,+ laya_shortlist,laya_status,validate_model,validate_preset,@@ -297,6 +298,186 @@ def test_private_contract():ok("contract/alias_english", router_agent(r, "English") is fake)+# --- shortlist tool (mocked router/agent, no weights) -------------------------++SHORTLIST_QUESTIONS = {+ "topic": {+ "type": "choice",+ "instructions": "Which topic?",+ "criteria": {+ "billing": "invoices and money",+ "shipping": "delivery status",+ "returns": "send items back",+ "account": "login and profile",+ "other": "anything else",+ },+ },+ "urgency": {"type": "score", "instructions": "How urgent?", "criteria": ["low", "high"]},+}+++class ShortlistAgent:+ device = "cpu"+++class ShortlistDirectAgent(ShortlistAgent):+ def __init__(self):+ self.seen = None++ def predict(self, state, questions, **kwargs):+ self.seen = questions+ return {"answers": {name: {"choice": list(spec["criteria"])[0], "confidence": 0.9}+ for name, spec in questions.items()}}+++class ShortlistRouter:+ """Fake router recording what predict received (no weights, no torch)."""++ def __init__(self, agents, routed="english"):+ self._agents = dict(agents)+ self._routed = routed+ self.seen_questions = None+ self.seen_kwargs = None++ def route(self, state, questions):+ return {"model": self._routed, "repo": "fake/repo", "reason": "unit-test route"}++ def predict(self, state, questions, **kwargs):+ self.seen_questions = questions+ self.seen_kwargs = kwargs+ answers = {}+ for name, spec in questions.items():+ if spec["type"] == "choice":+ labels = list(spec["criteria"])+ answers[name] = {"choice": labels[0], "confidence": 0.9,+ "probs": {label: round(1.0 / len(labels), 3) for label in labels}}+ elif spec["type"] == "score":+ answers[name] = {"score": 1.5, "confidence": 0.8, "distribution": [0.4, 0.6]}+ else:+ answers[name] = {"noul": 0.7, "confidence": 0.9}+ return {"answers": answers,+ "routing": {"model": kwargs.get("model", "english"), "repo": "fake/repo",+ "reason": "explicit model"}}+++class LoadingRouter(ShortlistRouter):+ """ShortlistRouter plus the on-demand load() Router.predict relies on."""++ def __init__(self, agents, routed="english"):+ super().__init__(agents, routed)+ self.load_calls = []++ def load(self, name):+ self.load_calls.append(name)+ agent = ShortlistAgent()+ self._agents[name] = agent+ return agent+++def _tie_embed(texts):+ # Every text gets the zero vector, so all cosine scores tie at 0 and the+ # documented tie rule ("ties keep the earlier label") keeps the first k.+ return [[0.0, 0.0] for _ in texts]+++def _raising_embed(texts):+ raise AssertionError("embed_fn must not be called when every choice passes through")+++def test_shortlist():+ expect_tool_error("shortlist/state_bad",+ lambda: laya_shortlist([], SHORTLIST_QUESTIONS), "invalid_state")+ expect_tool_error("shortlist/questions_bad",+ lambda: laya_shortlist(STATE, {}), "invalid_questions")+ expect_tool_error("shortlist/model_bad",+ lambda: laya_shortlist(STATE, SHORTLIST_QUESTIONS, model="gpt4"), "invalid_model")+ for bad_k in (0, -3, True, 2.5, "3"):+ expect_tool_error("shortlist/k_bad_%r" % (bad_k,),+ lambda b=bad_k: laya_shortlist(+ STATE, SHORTLIST_QUESTIONS, k=b,+ router=ShortlistRouter({"english": ShortlistAgent()})),+ "invalid_k")+ expect_tool_error("shortlist/auto_needs_router",+ lambda: laya_shortlist(STATE, SHORTLIST_QUESTIONS), "models_not_ready")+ # Explicit model whose checkpoint is neither resident nor loadable.+ expect_tool_error("shortlist/explicit_not_loaded",+ lambda: laya_shortlist(STATE, SHORTLIST_QUESTIONS, model="english",+ router=ShortlistRouter({})),+ "models_not_ready")++ # Passthrough: every choice has <= k labels, so embed_fn is never called.+ small = {"dept": {"type": "choice", "instructions": "pick", "criteria": {"a": "A", "b": "B"}}}+ router = ShortlistRouter({"english": ShortlistAgent()})+ out = laya_shortlist(STATE, small, model="english", k=5, router=router, embed_fn=_raising_embed)+ meta = out["shortlist"]["dept"]+ ok("shortlist/passthrough_meta", meta["passthrough"] is True and meta["scores"] is None+ and meta["k"] == 5 and meta["n"] == 2, repr(meta))+ ok("shortlist/passthrough_labels", meta["labels"] == ["a", "b"], repr(meta["labels"]))+ ok("shortlist/passthrough_answers", out["answers"]["dept"]["choice"] == "a", repr(out["answers"]))+ ok("shortlist/passthrough_forwarded", list(router.seen_questions["dept"]["criteria"]) == ["a", "b"])+ ok("shortlist/passthrough_device", out.get("device") == "cpu", repr(out.get("device")))+ ok("shortlist/passthrough_routing",+ out["routing"] == {"model": "english", "repo": None, "reason": "explicit model"},+ repr(out["routing"]))+ ok("shortlist/passthrough_latency", isinstance(out["latency_ms"], float))++ # Default k comes from laya.shortlist (20): a 3-option choice passes through.+ router = ShortlistRouter({"english": ShortlistAgent()})+ three = {"dept": {"type": "choice", "instructions": "pick",+ "criteria": {"a": "A", "b": "B", "c": "C"}}}+ out = laya_shortlist(STATE, three, model="english", router=router, embed_fn=_raising_embed)+ ok("shortlist/default_k_passthrough", out["shortlist"]["dept"]["k"] == 20+ and out["shortlist"]["dept"]["passthrough"] is True, repr(out["shortlist"]))++ # Shortlist path: 5 options with k=2 -> predict sees exactly the kept labels.+ calls = []++ def recording_embed(texts):+ calls.append(list(texts))+ return _tie_embed(texts)++ router = ShortlistRouter({"english": ShortlistAgent()})+ out = laya_shortlist(STATE, SHORTLIST_QUESTIONS, model="english", k=2,+ router=router, embed_fn=recording_embed)+ meta = out["shortlist"]["topic"]+ ok("shortlist/meta_shape", meta["k"] == 2 and meta["n"] == 5 and meta["passthrough"] is False,+ repr(meta))+ ok("shortlist/meta_labels_tie_order", meta["labels"] == ["billing", "shipping"], repr(meta["labels"]))+ ok("shortlist/meta_scores", meta["scores"] == [0.0, 0.0], repr(meta["scores"]))+ ok("shortlist/predict_saw_reduced",+ list(router.seen_questions["topic"]["criteria"]) == ["billing", "shipping"],+ repr(router.seen_questions["topic"]))+ ok("shortlist/embed_called_once", len(calls) == 1 and len(calls[0]) == 6,+ repr([len(c) for c in calls]))+ # Non-choice questions are forwarded unchanged and get no shortlist entry.+ ok("shortlist/non_choice_forwarded", router.seen_questions["urgency"] == SHORTLIST_QUESTIONS["urgency"])+ ok("shortlist/non_choice_no_meta", "urgency" not in out["shortlist"], repr(sorted(out["shortlist"])))+ ok("shortlist/input_not_mutated", len(SHORTLIST_QUESTIONS["topic"]["criteria"]) == 5)++ # Auto mode: route once, then answer with an explicit model= so the forward+ # pass does not re-route; the reported routing is the real route decision.+ router = ShortlistRouter({"multilingual": ShortlistAgent()}, routed="multilingual")+ out = laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, router=router, embed_fn=_tie_embed)+ ok("shortlist/auto_routing_reported",+ out["routing"] == {"model": "multilingual", "repo": "fake/repo", "reason": "unit-test route"},+ repr(out["routing"]))+ ok("shortlist/auto_explicit_model", router.seen_kwargs == {"model": "multilingual"},+ repr(router.seen_kwargs))++ # Lazy server (LAYA_PRELOAD=0): a routed checkpoint that is not resident is+ # loaded on demand, the same on-demand build Router.predict performs.+ router = LoadingRouter({}, routed="multilingual")+ out = laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, router=router, embed_fn=_tie_embed)+ ok("shortlist/lazy_load_once", router.load_calls == ["multilingual"], repr(router.load_calls))+ ok("shortlist/lazy_device", out.get("device") == "cpu", repr(out.get("device")))++ # Direct agent injection (no router), mirroring laya_predict's agent= path.+ agent = ShortlistDirectAgent()+ out = laya_shortlist(STATE, small, model="english", k=5, agent=agent, embed_fn=_raising_embed)+ ok("shortlist/direct_agent", out["answers"]["dept"]["choice"] == "a"+ and out["routing"]["reason"] == "explicit model", repr(out["routing"]))++def test_timeout_removed():# The per-call timeout was removed: a ThreadPoolExecutor shutdown waits for# the work anyway, and MCP clients apply their own request timeout. The tool@@ -305,7 +486,7 @@ def test_timeout_removed():import laya.mcp.tools as tools_mod- for fn in (laya_predict, laya_route, laya_preset):+ for fn in (laya_predict, laya_route, laya_preset, laya_shortlist):ok("timeout/param_absent_%s" % fn.__name__, "timeout" not in inspect.signature(fn).parameters)ok("timeout/executor_absent", "ThreadPoolExecutor" not in inspect.getsource(tools_mod))@@ -335,8 +516,8 @@ def test_models_from_env():def test_server_registration():tools = asyncio.run(mcp_server.list_tools())names = sorted(t.name for t in tools)- ok("server/tool_names", names == ["laya_predict", "laya_preset", "laya_route", "laya_status"], repr(names))- decision = {"laya_predict", "laya_route", "laya_preset"}+ ok("server/tool_names", names == ["laya_predict", "laya_preset", "laya_route", "laya_shortlist", "laya_status"], repr(names))+ decision = {"laya_predict", "laya_route", "laya_preset", "laya_shortlist"}for t in tools:desc = (t.description or "").lower()ok("server/desc_%s_nonempty" % t.name, bool(desc.strip()), repr(desc))@@ -351,6 +532,7 @@ test_real_device()test_private_contract()test_schema()test_shape()+test_shortlist()test_timeout_removed()test_models_from_env()test_server_registration()diff --git a/tests/test_mcp_local_e2e.py b/tests/test_mcp_local_e2e.pyindex 3bd1d8d1..6ccca90e 100644--- a/tests/test_mcp_local_e2e.py+++ b/tests/test_mcp_local_e2e.py@@ -2,7 +2,7 @@Launches ``python -m laya.mcp.server`` as a subprocess and speaks MCP overstdin/stdout (newline-delimited JSON-RPC), exactly like a real MCP client.-Exercises laya_predict, laya_preset, laya_route and laya_status against the+Exercises laya_predict, laya_preset, laya_route, laya_shortlist and laya_status against thelive checkpoints (downloaded via huggingface_hub on first run, cachedafterwards).@@ -115,7 +115,7 @@ def main():tools = client.request("tools/list", {})names = sorted(t["name"] for t in tools.get("tools", []))- ok("e2e/tool_names", names == ["laya_predict", "laya_preset", "laya_route", "laya_status"], repr(names))+ ok("e2e/tool_names", names == ["laya_predict", "laya_preset", "laya_route", "laya_shortlist", "laya_status"], repr(names))ticket = {"state": {@@ -165,6 +165,46 @@ def main():ok("e2e/route_model", payload.get("model") in ("english", "multilingual", "typed-decisions"), repr(payload))ok("e2e/route_reason", bool(payload.get("reason")), repr(payload))+ # laya_shortlist: k < n exercises the real embedding shortlist+ # (mean-pooled from the answering checkpoint's encoder) plus one+ # forward pass over the kept labels. Which two labels the+ # embeddings keep is checkpoint-dependent, so only the structural+ # contract is asserted: 2 kept labels, descending cosine scores,+ # and an answer drawn from the kept set.+ shortlist_call = {+ "state": ticket["state"],+ "questions": {+ "department": {+ "type": "choice",+ "instructions": "Which team should handle this ticket?",+ "criteria": {+ "billing": "payment, invoice, refund, duplicate charge",+ "technical": "bug, outage, integration problem",+ "account": "login, password, profile settings",+ },+ },+ },+ "k": 2,+ }+ result = client.request("tools/call", {"name": "laya_shortlist", "arguments": shortlist_call})+ payload = json.loads(result["content"][0]["text"])+ ok("e2e/shortlist_not_error", not result.get("isError"), repr(result.get("isError")))+ ans = (payload.get("answers") or {}).get("department") or {}+ meta = (payload.get("shortlist") or {}).get("department") or {}+ kept = meta.get("labels") or []+ scores = meta.get("scores") or []+ ok("e2e/shortlist_meta_shape", meta.get("k") == 2 and meta.get("n") == 3+ and meta.get("passthrough") is False and len(kept) == 2, repr(meta))+ ok("e2e/shortlist_scores_descending", len(scores) == 2+ and all(isinstance(s, float) for s in scores) and scores[0] >= scores[1],+ repr(scores))+ ok("e2e/shortlist_choice_in_kept", ans.get("choice") in kept,+ "choice=%r kept=%r" % (ans.get("choice"), kept))+ ok("e2e/shortlist_routing_model", (payload.get("routing") or {}).get("model")+ in ("english", "multilingual", "typed-decisions"), repr(payload.get("routing")))+ ok("e2e/shortlist_latency", 0 < payload.get("latency_ms", -1) < 60_000,+ repr(payload.get("latency_ms")))+result = client.request("tools/call", {"name": "laya_status", "arguments": {}})payload = json.loads(result["content"][0]["text"])if EXPECT_DEVICE: