Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(mcp): forward the routed language and reuse shortlist embeddings
Bruce-Yii0f7bf8e011da
diff --git a/laya/mcp/tools.py b/laya/mcp/tools.pyindex 810f5edb..afb31620 100644--- a/laya/mcp/tools.py+++ b/laya/mcp/tools.py@@ -196,6 +196,29 @@ def laya_route(state: Any, questions: Any, *, router: Any = None) -> dict:}+# Where the reusable shortlist embedder lives. Kept private and on the agent, so the+# cache's lifetime is the answering checkpoint's rather than this process's.+_EMBED_CACHE_ATTR = "_laya_mcp_shortlist_embed_fn"+++def _detected_language(detection: Any) -> str | None:+ """The language a route decision detected, or None when it reported none.++ Accepts either shape ``Router.route`` returns -- a ``RouteDecision`` (a dict)+ or a plain object -- and tolerates a decision that carries no detection at all,+ an explicit-model decision being the normal case.+ """+ if detection is None:+ return None+ if isinstance(detection, dict):+ lang = detection.get("language")+ else:+ lang = getattr(detection, "language", None)+ if isinstance(lang, str) and lang:+ return lang+ return None++def _resident_or_load(router: Any, name: str) -> Any:"""The resident agent for checkpoint ``name``, loading on demand when possible.@@ -213,6 +236,33 @@ def _resident_or_load(router: Any, name: str) -> Any:return load(name)+def _embed_fn_with_cache(agent: Any, embed_fn_from_agent: Any, cached_embed_fn: Any) -> Any:+ """An embedder for ``agent`` whose rows survive between shortlist calls.++ ``predict_shortlist`` embeds the query plus every option text on each call, so a+ fixed option list is re-encoded on every request even though it never changes.+ The ``cached_embed_fn`` wrapper from #405 fixes that, but only if the same+ wrapper is reused across calls.++ The wrapper is stored on the answering agent itself, so its lifetime is exactly+ that checkpoint's: another checkpoint, or the same checkpoint reloaded into a+ fresh ``Agent``, starts with an empty cache and can never reuse another+ checkpoint's rows. No module-level state is involved, and the cache stays bounded+ by ``cached_embed_fn``'s own LRU limit.+ """+ cached = getattr(agent, _EMBED_CACHE_ATTR, None)+ if cached is not None:+ return cached+ cached = cached_embed_fn(embed_fn_from_agent(agent))+ try:+ setattr(agent, _EMBED_CACHE_ATTR, cached)+ except (AttributeError, TypeError):+ # An agent that will not take attributes simply gets no reuse between calls,+ # which is the behaviour before this cache existed.+ pass+ return cached++def laya_shortlist(state: Any,questions: Any,@@ -236,7 +286,12 @@ def laya_shortlist(pass then runs with an explicit ``model=``, so routing happens once)."""# Lazy: keeps numpy/shortlist out of module import for laya.mcp.tools.- from laya.shortlist import DEFAULT_SHORTLIST_K, embed_fn_from_agent, predict_shortlist+ from laya.shortlist import (+ DEFAULT_SHORTLIST_K,+ cached_embed_fn,+ embed_fn_from_agent,+ predict_shortlist,+ )state_d = validate_state(state)questions_d = validate_questions(questions)@@ -255,6 +310,7 @@ def laya_shortlist(decision = router.route(state_d, questions_d)if isinstance(decision, dict):routed = decision.get("model")+ detection = decision.get("detection")routing = {"model": routed,"repo": decision.get("repo"),@@ -262,6 +318,7 @@ def laya_shortlist(}else:routed = getattr(decision, "model", None)+ detection = getattr(decision, "detection", None)routing = {"model": routed,"repo": getattr(decision, "repo", None),@@ -271,6 +328,14 @@ def laya_shortlist(raise ToolError("internal_error", "router.route returned no model")predict_target = routerpredict_kwargs: dict[str, Any] = {"model": routed}+ # Answering with an explicit model= makes predict re-route, and an+ # explicit-model decision carries no detection, so the language detected+ # here would otherwise be lost and the checkpoint's per-language+ # temperatures would not apply the way they do in laya_predict. Forward+ # it when the route actually reported one.+ lang = _detected_language(detection)+ if lang is not None:+ predict_kwargs["lang"] = langembed_agent = _resident_or_load(router, routed)elif agent is not None:routing = {"model": model_name, "repo": None, "reason": "explicit model"}@@ -287,7 +352,7 @@ def laya_shortlist(if embed_fn is None:try:- embed_fn = embed_fn_from_agent(embed_agent)+ embed_fn = _embed_fn_with_cache(embed_agent, embed_fn_from_agent, cached_embed_fn)except (AttributeError, TypeError, ValueError) as exc:raise ToolError("models_not_ready",diff --git a/tests/test_mcp.py b/tests/test_mcp.pyindex 4d3afab5..5804d751 100644--- a/tests/test_mcp.py+++ b/tests/test_mcp.py@@ -505,6 +505,156 @@ def test_shortlist():and out["routing"]["reason"] == "explicit model", repr(out["routing"]))+def test_shortlist_lang_parity():+ """Auto mode must forward the routed language, so per-language temperatures+ behave as they do in laya_predict. Passing an explicit model= makes+ Router.predict re-route with that model, and an explicit-model decision+ carries no detection -- so the language is lost unless shortlist passes it on.+ """+ class DetectingRouter(ShortlistRouter):+ def route(self, state, questions):+ return {+ "model": "multilingual",+ "repo": "fake/repo",+ "reason": "Latin script but language looks like 'de'",+ "detection": {"language": "de", "script": "Latin"},+ }++ router = DetectingRouter({"multilingual": ShortlistAgent()}, routed="multilingual")+ out = laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, router=router, embed_fn=_tie_embed)+ ok("shortlist_lang/auto_forwards_lang", (router.seen_kwargs or {}).get("lang") == "de",+ repr(router.seen_kwargs))+ ok("shortlist_lang/auto_still_explicit_model",+ (router.seen_kwargs or {}).get("model") == "multilingual", repr(router.seen_kwargs))+ ok("shortlist_lang/auto_routes_once",+ out["routing"]["model"] == "multilingual", repr(out["routing"]))++ # A decision without a usable language must not invent one.+ plain = ShortlistRouter({"english": ShortlistAgent()}, routed="english")+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, router=plain, embed_fn=_tie_embed)+ ok("shortlist_lang/no_detection_adds_nothing", plain.seen_kwargs == {"model": "english"},+ repr(plain.seen_kwargs))++ # Explicit-model mode is unchanged: no routing, so no language to forward.+ agent = ShortlistDirectAgent()+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="multilingual", agent=agent,+ embed_fn=_tie_embed)+ ok("shortlist_lang/explicit_model_untouched", agent.seen is not None)+++def test_shortlist_embed_cache():+ """A fixed option list must not be re-embedded on every call.++ Uses the answering checkpoint's own encoder (no injected embed_fn) through a+ weight-free fake tokenizer/encoder, so the number of tokenizer batches is a+ direct count of embedding work.+ """+ import torch++ class _Out:+ def __init__(self, hidden):+ self.last_hidden_state = hidden++ class CountingEncoder:+ def __init__(self):+ self.calls = 0++ def __call__(self, input_ids=None, attention_mask=None):+ self.calls += 1+ ids = input_ids.float()+ return _Out(torch.stack([ids, torch.ones_like(ids)], dim=-1))++ class CacheTok:+ def __init__(self):+ self.batches = []++ def __call__(self, texts, padding=True, truncation=True, max_length=256, return_tensors="pt"):+ self.batches.append(list(texts))+ rows = [[(ord(ch) % 5) + 1 for ch in t][:max_length] or [1] for t in texts]+ width = max(len(r) for r in rows)+ ids, mask = [], []+ for row in rows:+ pad = width - len(row)+ ids.append(row + [0] * pad)+ mask.append([1] * len(row) + [0] * pad)+ return {"input_ids": torch.tensor(ids, dtype=torch.long),+ "attention_mask": torch.tensor(mask, dtype=torch.long)}++ def make_agent():+ enc = CountingEncoder()+ tok = CacheTok()+ agent = ShortlistAgent()+ agent.tok = tok+ agent.encoder = enc+ agent.model = type("M", (), {"encoder": enc})()+ agent.device = torch.device("cpu")+ return agent++ # Same checkpoint, same option list: the second call re-embeds only the query.+ # Count embedded texts, not tokenizer batches: the whole set fits in one batch+ # either way, so batch count cannot tell the two cases apart.+ agent = make_agent()+ router = ShortlistRouter({"english": agent})++ def texts_since(mark):+ return sum(len(b) for b in agent.tok.batches[mark:])++ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="english", router=router)+ first_texts = texts_since(0)+ ok("shortlist_cache/first_call_embeds_query_and_options", first_texts == 6,+ "texts=%d batches=%r" % (first_texts, agent.tok.batches))++ # An identical repeat call embeds nothing at all: the query text is the same+ # too, and cached_embed_fn matches on exact strings.+ mark = len(agent.tok.batches)+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="english", router=router)+ ok("shortlist_cache/identical_repeat_embeds_nothing", texts_since(mark) == 0,+ "texts=%d batches=%r" % (texts_since(mark), agent.tok.batches[mark:]))++ # A different state is a new query, so a repeat with that state embeds the+ # query alone and reuses the five cached option rows.+ mark = len(agent.tok.batches)+ laya_shortlist({"text": "a completely different customer message"},+ SHORTLIST_QUESTIONS, k=2, model="english", router=router)+ ok("shortlist_cache/new_state_still_embedded", texts_since(mark) == 1,+ "texts=%d batches=%r" % (texts_since(mark), agent.tok.batches[mark:]))++ # A different answering checkpoint must not share another one's embeddings.+ other = make_agent()+ other_router = ShortlistRouter({"multilingual": other})+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="multilingual", router=other_router)+ ok("shortlist_cache/other_checkpoint_embeds_its_own",+ sum(len(b) for b in other.tok.batches) == 6,+ "texts=%d" % sum(len(b) for b in other.tok.batches))+ ok("shortlist_cache/other_checkpoint_encoder_used", other.encoder.calls == 1,+ "calls=%d" % other.encoder.calls)++ # Replacing the agent (a reload) starts a fresh cache rather than reusing rows.+ reloaded = make_agent()+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="english",+ router=ShortlistRouter({"english": reloaded}))+ ok("shortlist_cache/reloaded_agent_re_embeds",+ sum(len(b) for b in reloaded.tok.batches) == 6,+ "texts=%d" % sum(len(b) for b in reloaded.tok.batches))+ ok("shortlist_cache/other_checkpoint_still_cached",+ other.encoder.calls == 1, "calls=%d" % other.encoder.calls)++ # An injected embed_fn stays the caller's own, uncached.+ calls = []++ def counting_embed(texts):+ calls.append(list(texts))+ return [[0.0, 0.0] for _ in texts]++ inj_router = ShortlistRouter({"english": make_agent()})+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="english", router=inj_router,+ embed_fn=counting_embed)+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="english", router=inj_router,+ embed_fn=counting_embed)+ ok("shortlist_cache/injected_embed_not_wrapped", len(calls) == 2,+ "calls=%d" % len(calls))++def test_timeout_removed():# The per-call timeout was removed: a ThreadPoolExecutor shutdown waits for# the work anyway, and MCP clients apply their own request timeout. The tool@@ -563,6 +713,8 @@ test_schema()test_shape()test_question_forwarding()test_shortlist()+test_shortlist_lang_parity()+test_shortlist_embed_cache()test_timeout_removed()test_models_from_env()test_server_registration()