Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

mcp: add laya_shortlist tool for many-option choice questions

Aashish5df6910eb0f0
diff --git a/README.md b/README.md
index 507e230e..db591525 100644
--- a/README.md
+++ b/README.md
@@ -719,7 +719,7 @@ override on your own data rather than treating `A`/`B` as a universal fix.
Laya can be exposed as an [MCP](https://modelcontextprotocol.io) stdio server, so any MCP
client (OpenClaw, Claude Desktop, Cursor, ...) can call typed decisions as tools
-(`laya_predict`, `laya_route`, `laya_preset`, `laya_status`) without writing glue code.
+(`laya_predict`, `laya_route`, `laya_shortlist`, `laya_preset`, `laya_status`) without writing glue code.
This is an **optional extra**: the core package has no `mcp` dependency.
```bash
@@ -752,7 +752,13 @@ package:
| `LAYA_THREADS` | (torch default) | Same as `laya.serve`: cap torch intra-op threads for CPU inference; keep it at or below the physical core count |
The tools return structured JSON (answers with probabilities, routing metadata, device,
-`latency_ms`). As with the SDK, use it for structured decisions only; not for open Q&A or
+`latency_ms`). `laya_shortlist` is the MCP form of [`predict_shortlist`](#honest-limits):
+it shortlists a many-option choice question to its `k` most likely labels by embedding
+similarity (mean-pooled from the answering checkpoint's own encoder, so no extra model is
+downloaded), answers in one forward pass, and returns per-question shortlist metadata
+(kept labels, cosine scores, `k`, option count). The guardrails shown on every decision
+tool point clients here for >20-option choices. As with the SDK, use it for structured
+decisions only; not for open Q&A or
text generation. Tests: `tests/test_mcp.py` (CI, no weights) and
`tests/test_mcp_local_e2e.py` (local, real weights and a real stdio handshake).
diff --git a/laya/mcp/server.py b/laya/mcp/server.py
index eea0847a..d76e67f7 100644
--- a/laya/mcp/server.py
+++ b/laya/mcp/server.py
@@ -42,6 +42,7 @@ from .tools import (
laya_predict,
laya_preset,
laya_route,
+ laya_shortlist,
laya_status,
)
@@ -69,7 +70,7 @@ _GUARDRAILS = (
"noul (calibrated P(true)). One forward pass ~33ms (GPU) / ~200ms (CPU). "
"No text generation, so no hallucination. Do NOT use for open Q&A, summarization, "
"rewriting, code, or multi-hop reasoning. Do NOT use for >20-option choice "
- "questions without shortlisting."
+ "questions without shortlisting (use the laya_shortlist tool)."
)
@@ -197,6 +198,34 @@ def laya_predict_tool(state: dict, questions: dict, model: str = "auto") -> str:
)
+ name="laya_shortlist",
+ description=(
+ "Shortlist a many-option choice question to its k most likely labels by embedding "
+ "similarity (mean-pooled from the answering checkpoint's own encoder, so no extra "
+ "model is downloaded), then answer in one forward pass. Use this instead of "
+ "laya_predict whenever a choice question has more options than the guardrails allow. "
+ "Returns the answers plus per-question shortlist metadata (kept labels, cosine "
+ "scores, k, option count). "
+ + _GUARDRAILS
+ ),
+)
+def laya_shortlist_tool(state: dict, questions: dict, model: str = "auto", k: int = 20) -> str:
+ """Shortlist many-option choice questions, then answer."""
+ # k's default mirrors laya.shortlist.DEFAULT_SHORTLIST_K; it is a literal
+ # here so the MCP schema carries the default without importing numpy at
+ # server start.
+ router = _router_or_error()
+ return _wrap(
+ laya_shortlist,
+ state=state,
+ questions=questions,
+ model=model,
+ k=k,
+ router=router,
+ )
+
+
@server.tool(
name="laya_preset",
description=(
diff --git a/laya/mcp/tools.py b/laya/mcp/tools.py
index b3fc6b74..0e05be8a 100644
--- a/laya/mcp/tools.py
+++ b/laya/mcp/tools.py
@@ -3,7 +3,7 @@
from __future__ import annotations
import time
-from typing import Any, Callable, Protocol
+from typing import Any, Callable, Protocol, Sequence
from .device import agent_device, device_report, router_agent
@@ -194,6 +194,124 @@ def laya_route(state: Any, questions: Any, *, router: Any = None) -> dict:
}
+def _resident_or_load(router: Any, name: str) -> Any:
+ """The resident agent for checkpoint ``name``, loading on demand when possible.
+
+ Read-only lookup first: ``router_agent`` never calls ``load()``. A miss
+ falls back to ``Router.load``, the same on-demand build ``Router.predict``
+ performs after routing, so a lazily preloaded server (LAYA_PRELOAD=0, or a
+ checkpoint outside LAYA_MODELS) behaves exactly like ``laya_predict``.
+ """
+ resident = router_agent(router, name)
+ if resident is not None:
+ return resident
+ load = getattr(router, "load", None)
+ if load is None:
+ raise ToolError("models_not_ready", f"checkpoint {name!r} is not loaded")
+ return load(name)
+
+
+def laya_shortlist(
+ state: Any,
+ questions: Any,
+ model: Any = "auto",
+ k: Any = None,
+ *,
+ router: Any = None,
+ agent: Any = None,
+ embed_fn: Callable[[Sequence[str]], Any] | None = None,
+) -> dict:
+ """Shortlist many-option choice questions to ``k`` labels, then one predict.
+
+ The shared guardrails tell clients not to run >20-option choice questions
+ without shortlisting; this tool is that shortlisting (the in-process
+ ``laya.shortlist.predict_shortlist`` pattern over MCP). Embeddings come
+ from the answering checkpoint's own encoder (``embed_fn_from_agent``), so
+ no extra model is downloaded; ``embed_fn`` is injectable for tests or for
+ a dedicated bi-encoder.
+
+ ``routing`` reports the real route decision in auto mode (the forward
+ pass then runs with an explicit ``model=``, so routing happens once).
+ """
+ # Lazy: keeps numpy/shortlist out of module import for laya.mcp.tools.
+ from laya.shortlist import DEFAULT_SHORTLIST_K, embed_fn_from_agent, predict_shortlist
+
+ state_d = validate_state(state)
+ questions_d = validate_questions(questions)
+ model_name = validate_model(model)
+ if k is None:
+ k = DEFAULT_SHORTLIST_K
+ if isinstance(k, bool) or not isinstance(k, int) or k < 1:
+ raise ToolError("invalid_k", f"k must be a positive integer, got {k!r}")
+
+ routing: dict[str, Any]
+ if model_name == "auto":
+ if router is None:
+ raise ToolError("models_not_ready", "Router is not loaded (auto mode)")
+ if not hasattr(router, "route"):
+ raise ToolError("internal_error", "router has no route() method")
+ decision = router.route(state_d, questions_d)
+ if isinstance(decision, dict):
+ routed = decision.get("model")
+ routing = {
+ "model": routed,
+ "repo": decision.get("repo"),
+ "reason": decision.get("reason"),
+ }
+ else:
+ routed = getattr(decision, "model", None)
+ routing = {
+ "model": routed,
+ "repo": getattr(decision, "repo", None),
+ "reason": getattr(decision, "reason", None),
+ }
+ if not isinstance(routed, str) or not routed:
+ raise ToolError("internal_error", "router.route returned no model")
+ predict_target = router
+ predict_kwargs: dict[str, Any] = {"model": routed}
+ embed_agent = _resident_or_load(router, routed)
+ elif agent is not None:
+ routing = {"model": model_name, "repo": None, "reason": "explicit model"}
+ predict_target = agent
+ predict_kwargs = {}
+ embed_agent = agent
+ else:
+ if router is None:
+ raise ToolError("models_not_ready", "no agent/router loaded")
+ routing = {"model": model_name, "repo": None, "reason": "explicit model"}
+ predict_target = router
+ predict_kwargs = {"model": model_name}
+ embed_agent = _resident_or_load(router, model_name)
+
+ if embed_fn is None:
+ try:
+ embed_fn = embed_fn_from_agent(embed_agent)
+ except (AttributeError, TypeError, ValueError) as exc:
+ raise ToolError(
+ "models_not_ready",
+ f"cannot build shortlist embeddings from checkpoint {routing['model']!r}: {exc}",
+ ) from exc
+
+ started = time.perf_counter()
+ result = predict_shortlist(predict_target, state_d, questions_d, embed_fn, k=k, **predict_kwargs)
+ latency_ms = (time.perf_counter() - started) * 1000.0
+
+ if not isinstance(result, dict):
+ raise ToolError("internal_error", "predict returned non-object")
+ answers = _normalize_answers(result["answers"])
+ # The answering checkpoint is the embedding checkpoint in every branch.
+ device = agent_device(embed_agent)
+ out: dict[str, Any] = {
+ "answers": answers,
+ "routing": routing,
+ "shortlist": result.get("shortlist") or {},
+ "latency_ms": round(latency_ms, 3),
+ }
+ if device:
+ out["device"] = device
+ return out
+
+
def laya_preset(
preset: Any,
state: Any,
diff --git a/tests/test_mcp.py b/tests/test_mcp.py
index b290fec6..da6ebdee 100644
--- a/tests/test_mcp.py
+++ b/tests/test_mcp.py
@@ -32,6 +32,7 @@ from laya.mcp.tools import ( # noqa: E402
laya_predict,
laya_preset,
laya_route,
+ laya_shortlist,
laya_status,
validate_model,
validate_preset,
@@ -297,6 +298,186 @@ def test_private_contract():
ok("contract/alias_english", router_agent(r, "English") is fake)
+# --- shortlist tool (mocked router/agent, no weights) -------------------------
+
+SHORTLIST_QUESTIONS = {
+ "topic": {
+ "type": "choice",
+ "instructions": "Which topic?",
+ "criteria": {
+ "billing": "invoices and money",
+ "shipping": "delivery status",
+ "returns": "send items back",
+ "account": "login and profile",
+ "other": "anything else",
+ },
+ },
+ "urgency": {"type": "score", "instructions": "How urgent?", "criteria": ["low", "high"]},
+}
+
+
+class ShortlistAgent:
+ device = "cpu"
+
+
+class ShortlistDirectAgent(ShortlistAgent):
+ def __init__(self):
+ self.seen = None
+
+ def predict(self, state, questions, **kwargs):
+ self.seen = questions
+ return {"answers": {name: {"choice": list(spec["criteria"])[0], "confidence": 0.9}
+ for name, spec in questions.items()}}
+
+
+class ShortlistRouter:
+ """Fake router recording what predict received (no weights, no torch)."""
+
+ def __init__(self, agents, routed="english"):
+ self._agents = dict(agents)
+ self._routed = routed
+ self.seen_questions = None
+ self.seen_kwargs = None
+
+ def route(self, state, questions):
+ return {"model": self._routed, "repo": "fake/repo", "reason": "unit-test route"}
+
+ def predict(self, state, questions, **kwargs):
+ self.seen_questions = questions
+ self.seen_kwargs = kwargs
+ answers = {}
+ for name, spec in questions.items():
+ if spec["type"] == "choice":
+ labels = list(spec["criteria"])
+ answers[name] = {"choice": labels[0], "confidence": 0.9,
+ "probs": {label: round(1.0 / len(labels), 3) for label in labels}}
+ elif spec["type"] == "score":
+ answers[name] = {"score": 1.5, "confidence": 0.8, "distribution": [0.4, 0.6]}
+ else:
+ answers[name] = {"noul": 0.7, "confidence": 0.9}
+ return {"answers": answers,
+ "routing": {"model": kwargs.get("model", "english"), "repo": "fake/repo",
+ "reason": "explicit model"}}
+
+
+class LoadingRouter(ShortlistRouter):
+ """ShortlistRouter plus the on-demand load() Router.predict relies on."""
+
+ def __init__(self, agents, routed="english"):
+ super().__init__(agents, routed)
+ self.load_calls = []
+
+ def load(self, name):
+ self.load_calls.append(name)
+ agent = ShortlistAgent()
+ self._agents[name] = agent
+ return agent
+
+
+def _tie_embed(texts):
+ # Every text gets the zero vector, so all cosine scores tie at 0 and the
+ # documented tie rule ("ties keep the earlier label") keeps the first k.
+ return [[0.0, 0.0] for _ in texts]
+
+
+def _raising_embed(texts):
+ raise AssertionError("embed_fn must not be called when every choice passes through")
+
+
+def test_shortlist():
+ expect_tool_error("shortlist/state_bad",
+ lambda: laya_shortlist([], SHORTLIST_QUESTIONS), "invalid_state")
+ expect_tool_error("shortlist/questions_bad",
+ lambda: laya_shortlist(STATE, {}), "invalid_questions")
+ expect_tool_error("shortlist/model_bad",
+ lambda: laya_shortlist(STATE, SHORTLIST_QUESTIONS, model="gpt4"), "invalid_model")
+ for bad_k in (0, -3, True, 2.5, "3"):
+ expect_tool_error("shortlist/k_bad_%r" % (bad_k,),
+ lambda b=bad_k: laya_shortlist(
+ STATE, SHORTLIST_QUESTIONS, k=b,
+ router=ShortlistRouter({"english": ShortlistAgent()})),
+ "invalid_k")
+ expect_tool_error("shortlist/auto_needs_router",
+ lambda: laya_shortlist(STATE, SHORTLIST_QUESTIONS), "models_not_ready")
+ # Explicit model whose checkpoint is neither resident nor loadable.
+ expect_tool_error("shortlist/explicit_not_loaded",
+ lambda: laya_shortlist(STATE, SHORTLIST_QUESTIONS, model="english",
+ router=ShortlistRouter({})),
+ "models_not_ready")
+
+ # Passthrough: every choice has <= k labels, so embed_fn is never called.
+ small = {"dept": {"type": "choice", "instructions": "pick", "criteria": {"a": "A", "b": "B"}}}
+ router = ShortlistRouter({"english": ShortlistAgent()})
+ out = laya_shortlist(STATE, small, model="english", k=5, router=router, embed_fn=_raising_embed)
+ meta = out["shortlist"]["dept"]
+ ok("shortlist/passthrough_meta", meta["passthrough"] is True and meta["scores"] is None
+ and meta["k"] == 5 and meta["n"] == 2, repr(meta))
+ ok("shortlist/passthrough_labels", meta["labels"] == ["a", "b"], repr(meta["labels"]))
+ ok("shortlist/passthrough_answers", out["answers"]["dept"]["choice"] == "a", repr(out["answers"]))
+ ok("shortlist/passthrough_forwarded", list(router.seen_questions["dept"]["criteria"]) == ["a", "b"])
+ ok("shortlist/passthrough_device", out.get("device") == "cpu", repr(out.get("device")))
+ ok("shortlist/passthrough_routing",
+ out["routing"] == {"model": "english", "repo": None, "reason": "explicit model"},
+ repr(out["routing"]))
+ ok("shortlist/passthrough_latency", isinstance(out["latency_ms"], float))
+
+ # Default k comes from laya.shortlist (20): a 3-option choice passes through.
+ router = ShortlistRouter({"english": ShortlistAgent()})
+ three = {"dept": {"type": "choice", "instructions": "pick",
+ "criteria": {"a": "A", "b": "B", "c": "C"}}}
+ out = laya_shortlist(STATE, three, model="english", router=router, embed_fn=_raising_embed)
+ ok("shortlist/default_k_passthrough", out["shortlist"]["dept"]["k"] == 20
+ and out["shortlist"]["dept"]["passthrough"] is True, repr(out["shortlist"]))
+
+ # Shortlist path: 5 options with k=2 -> predict sees exactly the kept labels.
+ calls = []
+
+ def recording_embed(texts):
+ calls.append(list(texts))
+ return _tie_embed(texts)
+
+ router = ShortlistRouter({"english": ShortlistAgent()})
+ out = laya_shortlist(STATE, SHORTLIST_QUESTIONS, model="english", k=2,
+ router=router, embed_fn=recording_embed)
+ meta = out["shortlist"]["topic"]
+ ok("shortlist/meta_shape", meta["k"] == 2 and meta["n"] == 5 and meta["passthrough"] is False,
+ repr(meta))
+ ok("shortlist/meta_labels_tie_order", meta["labels"] == ["billing", "shipping"], repr(meta["labels"]))
+ ok("shortlist/meta_scores", meta["scores"] == [0.0, 0.0], repr(meta["scores"]))
+ ok("shortlist/predict_saw_reduced",
+ list(router.seen_questions["topic"]["criteria"]) == ["billing", "shipping"],
+ repr(router.seen_questions["topic"]))
+ ok("shortlist/embed_called_once", len(calls) == 1 and len(calls[0]) == 6,
+ repr([len(c) for c in calls]))
+ # Non-choice questions are forwarded unchanged and get no shortlist entry.
+ ok("shortlist/non_choice_forwarded", router.seen_questions["urgency"] == SHORTLIST_QUESTIONS["urgency"])
+ ok("shortlist/non_choice_no_meta", "urgency" not in out["shortlist"], repr(sorted(out["shortlist"])))
+ ok("shortlist/input_not_mutated", len(SHORTLIST_QUESTIONS["topic"]["criteria"]) == 5)
+
+ # Auto mode: route once, then answer with an explicit model= so the forward
+ # pass does not re-route; the reported routing is the real route decision.
+ router = ShortlistRouter({"multilingual": ShortlistAgent()}, routed="multilingual")
+ out = laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, router=router, embed_fn=_tie_embed)
+ ok("shortlist/auto_routing_reported",
+ out["routing"] == {"model": "multilingual", "repo": "fake/repo", "reason": "unit-test route"},
+ repr(out["routing"]))
+ ok("shortlist/auto_explicit_model", router.seen_kwargs == {"model": "multilingual"},
+ repr(router.seen_kwargs))
+
+ # Lazy server (LAYA_PRELOAD=0): a routed checkpoint that is not resident is
+ # loaded on demand, the same on-demand build Router.predict performs.
+ router = LoadingRouter({}, routed="multilingual")
+ out = laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, router=router, embed_fn=_tie_embed)
+ ok("shortlist/lazy_load_once", router.load_calls == ["multilingual"], repr(router.load_calls))
+ ok("shortlist/lazy_device", out.get("device") == "cpu", repr(out.get("device")))
+
+ # Direct agent injection (no router), mirroring laya_predict's agent= path.
+ agent = ShortlistDirectAgent()
+ out = laya_shortlist(STATE, small, model="english", k=5, agent=agent, embed_fn=_raising_embed)
+ ok("shortlist/direct_agent", out["answers"]["dept"]["choice"] == "a"
+ and out["routing"]["reason"] == "explicit model", repr(out["routing"]))
+
+
def test_timeout_removed():
# The per-call timeout was removed: a ThreadPoolExecutor shutdown waits for
# the work anyway, and MCP clients apply their own request timeout. The tool
@@ -305,7 +486,7 @@ def test_timeout_removed():
import laya.mcp.tools as tools_mod
- for fn in (laya_predict, laya_route, laya_preset):
+ for fn in (laya_predict, laya_route, laya_preset, laya_shortlist):
ok("timeout/param_absent_%s" % fn.__name__, "timeout" not in inspect.signature(fn).parameters)
ok("timeout/executor_absent", "ThreadPoolExecutor" not in inspect.getsource(tools_mod))
@@ -335,8 +516,8 @@ def test_models_from_env():
def test_server_registration():
tools = asyncio.run(mcp_server.list_tools())
names = sorted(t.name for t in tools)
- ok("server/tool_names", names == ["laya_predict", "laya_preset", "laya_route", "laya_status"], repr(names))
- decision = {"laya_predict", "laya_route", "laya_preset"}
+ ok("server/tool_names", names == ["laya_predict", "laya_preset", "laya_route", "laya_shortlist", "laya_status"], repr(names))
+ decision = {"laya_predict", "laya_route", "laya_preset", "laya_shortlist"}
for t in tools:
desc = (t.description or "").lower()
ok("server/desc_%s_nonempty" % t.name, bool(desc.strip()), repr(desc))
@@ -351,6 +532,7 @@ test_real_device()
test_private_contract()
test_schema()
test_shape()
+test_shortlist()
test_timeout_removed()
test_models_from_env()
test_server_registration()
diff --git a/tests/test_mcp_local_e2e.py b/tests/test_mcp_local_e2e.py
index 3bd1d8d1..6ccca90e 100644
--- a/tests/test_mcp_local_e2e.py
+++ b/tests/test_mcp_local_e2e.py
@@ -2,7 +2,7 @@
Launches ``python -m laya.mcp.server`` as a subprocess and speaks MCP over
stdin/stdout (newline-delimited JSON-RPC), exactly like a real MCP client.
-Exercises laya_predict, laya_preset, laya_route and laya_status against the
+Exercises laya_predict, laya_preset, laya_route, laya_shortlist and laya_status against the
live checkpoints (downloaded via huggingface_hub on first run, cached
afterwards).
@@ -115,7 +115,7 @@ def main():
tools = client.request("tools/list", {})
names = sorted(t["name"] for t in tools.get("tools", []))
- ok("e2e/tool_names", names == ["laya_predict", "laya_preset", "laya_route", "laya_status"], repr(names))
+ ok("e2e/tool_names", names == ["laya_predict", "laya_preset", "laya_route", "laya_shortlist", "laya_status"], repr(names))
ticket = {
"state": {
@@ -165,6 +165,46 @@ def main():
ok("e2e/route_model", payload.get("model") in ("english", "multilingual", "typed-decisions"), repr(payload))
ok("e2e/route_reason", bool(payload.get("reason")), repr(payload))
+ # laya_shortlist: k < n exercises the real embedding shortlist
+ # (mean-pooled from the answering checkpoint's encoder) plus one
+ # forward pass over the kept labels. Which two labels the
+ # embeddings keep is checkpoint-dependent, so only the structural
+ # contract is asserted: 2 kept labels, descending cosine scores,
+ # and an answer drawn from the kept set.
+ shortlist_call = {
+ "state": ticket["state"],
+ "questions": {
+ "department": {
+ "type": "choice",
+ "instructions": "Which team should handle this ticket?",
+ "criteria": {
+ "billing": "payment, invoice, refund, duplicate charge",
+ "technical": "bug, outage, integration problem",
+ "account": "login, password, profile settings",
+ },
+ },
+ },
+ "k": 2,
+ }
+ result = client.request("tools/call", {"name": "laya_shortlist", "arguments": shortlist_call})
+ payload = json.loads(result["content"][0]["text"])
+ ok("e2e/shortlist_not_error", not result.get("isError"), repr(result.get("isError")))
+ ans = (payload.get("answers") or {}).get("department") or {}
+ meta = (payload.get("shortlist") or {}).get("department") or {}
+ kept = meta.get("labels") or []
+ scores = meta.get("scores") or []
+ ok("e2e/shortlist_meta_shape", meta.get("k") == 2 and meta.get("n") == 3
+ and meta.get("passthrough") is False and len(kept) == 2, repr(meta))
+ ok("e2e/shortlist_scores_descending", len(scores) == 2
+ and all(isinstance(s, float) for s in scores) and scores[0] >= scores[1],
+ repr(scores))
+ ok("e2e/shortlist_choice_in_kept", ans.get("choice") in kept,
+ "choice=%r kept=%r" % (ans.get("choice"), kept))
+ ok("e2e/shortlist_routing_model", (payload.get("routing") or {}).get("model")
+ in ("english", "multilingual", "typed-decisions"), repr(payload.get("routing")))
+ ok("e2e/shortlist_latency", 0 < payload.get("latency_ms", -1) < 60_000,
+ repr(payload.get("latency_ms")))
+
result = client.request("tools/call", {"name": "laya_status", "arguments": {}})
payload = json.loads(result["content"][0]["text"])
if EXPECT_DEVICE: