Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #565 from Bruce-Yii/fix/mcp-question-validation

Nandakishor9a5c2539da76
diff --cc laya/mcp/tools.py
index bf3af58b,ee2d3dfc..ee294383
--- a/laya/mcp/tools.py
+++ b/laya/mcp/tools.py
@@@ -27,55 -25,46 +27,89 @@@ PRESETS: dict[str, str] =
"moderation": "moderation_questions",
"triage": "triage_questions",
"model_router": "router_questions",
+ "email": "email_questions",
}
+# `router` is what the CLI calls this preset, and it is the word a caller reaches for first;
+# `model_router` stayed because clients already have it in their prompts. Both name one preset,
+# and the canonical key is what comes back in the result.
+PRESET_ALIASES: dict[str, str] = {"router": "model_router"}
+
+
+def get_available_presets() -> dict[str, dict[str, Any]]:
+ """The built-in presets a ``laya_preset`` call can name, with what each one reads.
+
+ The state field comes from the questions themselves rather than from a table beside them, so
+ this cannot drift from the presets: a caller that has to hand-build a state (and the MCP
+ ``tools/list`` description, which is built from here) sees the field each preset actually asks
+ about. Builders are resolved through ``laya`` at call time, the way ``laya.mcp.server`` resolves
+ them, so a preset added in a newer version shows up here as soon as it is importable.
+ """
+ import laya
+
+ aliases: dict[str, list[str]] = {}
+ for alias, target in PRESET_ALIASES.items():
+ aliases.setdefault(target, []).append(alias)
+
+ out: dict[str, dict[str, Any]] = {}
+ for name, attr in sorted(PRESETS.items()):
+ entry: dict[str, Any] = {"questions": attr}
+ if name in aliases:
+ entry["aliases"] = sorted(aliases[name])
+ builder = getattr(laya, attr, None)
+ if builder is not None:
+ questions = builder()
+ entry["n_questions"] = len(questions)
+ field = state_field(questions)
+ if field is not None:
+ entry["state_field"] = field
+ out[name] = entry
+ return out
+
+
VALID_TYPES = {"choice", "score", "noul"}
-VALID_MODELS = {"auto", "english", "multilingual", "typed-decisions"}
+# `auto` is this layer's own sentinel -- "route it, do not pin a checkpoint". Every other value is a
+# checkpoint name, and the registry of those (names, aliases, casing) is core's: `laya.router` runs
+# every model argument through `normalise_name` before it loads anything, which is the same call the
+# tools below end up making through `router.predict(model=...)`. A second list here could only ever
+# be narrower than that one, so it is not repeated.
+AUTO = "auto"
+ def _check_noul_labels(name: str, labels: Any) -> None:
+ """`labels` renames a noul answer's two model-facing texts; mirror the agent's rule.
+
+ `laya.common._resolve_noul_labels` is the contract: exactly `false` and `true`, each a
+ distinct non-empty string. The agent raises `ValueError` on anything else, and the tool
+ wrapper turns a `ValueError` into `internal_error` -- the code this layer uses for a tool
+ that broke. So the check is repeated here rather than imported, both to keep the message
+ in this layer's `questions[...]` voice and to keep `laya.mcp.tools` free of a torch import.
+ """
+ if not isinstance(labels, dict) or set(labels) != {"false", "true"}:
+ raise ToolError(
+ "invalid_questions",
+ f"questions[{name}].labels must map exactly 'false' and 'true' to distinct "
+ f"non-empty strings",
+ )
+ values = [labels["false"], labels["true"]]
+ # The agent requires a str here, and it is checked before the text is used: a number or a
+ # bool is not a label, it is a type mistake. Stringifying first would quietly accept what
+ # the agent rejects, and the rejection would then arrive too late to be reported as a
+ # caller error.
+ if not all(isinstance(value, str) for value in values):
+ raise ToolError(
+ "invalid_questions",
+ f"questions[{name}].labels must give 'false' and 'true' string values, got "
+ f"{[type(v).__name__ for v in values]}",
+ )
+ texts = [value.strip() for value in values]
+ if not texts[0] or not texts[1] or texts[0] == texts[1]:
+ raise ToolError(
+ "invalid_questions",
+ f"questions[{name}].labels must give 'false' and 'true' distinct non-empty strings",
+ )
+
+
def validate_questions(questions: Any) -> dict:
if not isinstance(questions, dict) or not questions:
raise ToolError(
diff --cc tests/test_mcp.py
index ce4b49ae,9a28cb8b..44e27f6e
--- a/tests/test_mcp.py
+++ b/tests/test_mcp.py
@@@ -721,802 -514,143 +730,939 @@@ def test_shortlist()
and out["routing"]["reason"] == "explicit model", repr(out["routing"]))
+# --- batch tools (mocked router, no weights) ----------------------------------
+
+class BatchRouter(FakeRouter):
+ """Records each batch call; returns one result per request, input order."""
+
+ def __init__(self):
+ self.predict_batch_calls = []
+ self.route_batch_calls = []
+
+ def _answer_for(self, request):
+ answers = {}
+ for name, spec in request["questions"].items():
+ if spec["type"] == "choice":
+ answers[name] = {"choice": "billing", "confidence": 0.9}
+ elif spec["type"] == "score":
+ answers[name] = {"score": 1.5, "confidence": 0.8}
+ else:
+ answers[name] = {"noul": 0.7, "confidence": 0.9}
+ return {"answers": answers,
+ "routing": {"model": request.get("model") or "english",
+ "repo": "fake/laya", "reason": "batch route"}}
+
+ def predict_batch(self, requests, batch_size=None):
+ self.predict_batch_calls.append((list(requests), batch_size))
+ return [self._answer_for(request) for request in requests]
+
+ def route_batch(self, requests):
+ self.route_batch_calls.append(list(requests))
+ return [{"model": request.get("model") or "english", "repo": "fake/laya",
+ "reason": "batch route"} for request in requests]
+
+
+class ShortRouter:
+ """Router whose batches misreport their own size."""
+
+ def predict_batch(self, requests, batch_size=None):
+ return []
+
+ def route_batch(self, requests):
+ return []
+
+
+BATCH_REQUESTS = [
+ {"state": {"body": "refund please"}, "questions": QUESTIONS},
+ {"state": {"body": "please refund invoice 2"}, "questions": QUESTIONS,
+ "model": "english", "task": "massive", "lang": "en"},
+ {"state": {"body": "mera account double charge hua"},
+ "questions": {"triage": {"type": "noul", "instructions": "Leaving?"}},
+ "lang": "hi"},
+]
+
+
+def test_batch_validation():
+ for bad in (None, {}, "x", {"state": STATE, "questions": QUESTIONS}):
+ expect_tool_error("batch/requests_bad_%r" % (type(bad).__name__,),
+ lambda b=bad: validate_batch_requests(b), "invalid_request")
+ expect_tool_error("batch/requests_empty",
+ lambda: validate_batch_requests([]), "invalid_request")
+ expect_tool_error("batch/item_not_object",
+ lambda: validate_batch_requests(["x"]), "invalid_request")
+ expect_tool_error("batch/item_missing_state",
+ lambda: validate_batch_requests([{"questions": QUESTIONS}]), "invalid_state")
+ expect_tool_error("batch/item_missing_questions",
+ lambda: validate_batch_requests([{"state": STATE}]), "invalid_questions")
+ expect_tool_error("batch/item_bad_questions",
+ lambda: validate_batch_requests([{"state": STATE, "questions": {}}]),
+ "invalid_questions")
+ expect_tool_error("batch/item_bad_model",
+ lambda: validate_batch_requests(
+ [{"state": STATE, "questions": QUESTIONS, "model": "gpt4"}]),
+ "invalid_model")
+ for key, value in (("task", 3), ("lang", ""), ("lang", [])):
+ expect_tool_error("batch/item_bad_%s" % key,
+ lambda k=key, v=value: validate_batch_requests(
+ [{"state": STATE, "questions": QUESTIONS, k: v}]),
+ "invalid_request")
+
+ out = validate_batch_requests(BATCH_REQUESTS)
+ ok("batch/validation_preserves_order", [item["state"] for item in out]
+ == [request["state"] for request in BATCH_REQUESTS])
+ ok("batch/validation_keeps_overrides",
+ out[1].get("model") == "english" and out[1].get("task") == "massive"
+ and out[1].get("lang") == "en" and out[2].get("lang") == "hi", repr(out[1]))
+ # "auto" is the same thing as absent: Router.route resolves the checkpoint.
+ out = validate_batch_requests([{"state": STATE, "questions": QUESTIONS, "model": "auto"}])
+ ok("batch/validation_auto_dropped", "model" not in out[0], repr(out[0]))
+ # Keys the Router would never expect are dropped, not forwarded.
+ out = validate_batch_requests([{"state": STATE, "questions": QUESTIONS, "temperature": 0}])
+ ok("batch/validation_unknown_key_dropped", set(out[0]) == {"state", "questions"}, repr(out[0]))
+
+
+def test_batch_predict():
+ expect_tool_error("batch/predict_no_router",
+ lambda: laya_predict_batch(BATCH_REQUESTS, router=None), "models_not_ready")
+ expect_tool_error("batch/predict_no_method",
+ lambda: laya_predict_batch(BATCH_REQUESTS, router=FakeRouter()),
+ "internal_error")
+ for bad_size in (True, 0, -2, 2.5, "3"):
+ expect_tool_error("batch/predict_bad_size_%r" % (bad_size,),
+ lambda s=bad_size: laya_predict_batch(
+ BATCH_REQUESTS, batch_size=s, router=BatchRouter()),
+ "invalid_batch_size")
+ # Validation runs before the router is touched: one bad item, no partial batch.
+ router = BatchRouter()
+ expect_tool_error("batch/predict_validates_first",
+ lambda: laya_predict_batch([{"state": STATE}, {"state": STATE, "questions": QUESTIONS}],
+ router=router),
+ "invalid_questions")
+ ok("batch/predict_not_called_on_bad_input", router.predict_batch_calls == [])
+
+ router = BatchRouter()
+ out = laya_predict_batch(BATCH_REQUESTS, batch_size=8, router=router)
+ ok("batch/predict_one_call", len(router.predict_batch_calls) == 1,
+ repr(len(router.predict_batch_calls)))
+ forwarded, size = router.predict_batch_calls[0]
+ ok("batch/predict_size_forwarded", size == 8, repr(size))
+ ok("batch/predict_items_forwarded", [item["state"] for item in forwarded]
+ == [request["state"] for request in BATCH_REQUESTS])
+
+ ok("batch/predict_keys", set(out) == {"requests", "model_counts",
+ "total_latency_ms", "per_request_latency_ms"}, repr(sorted(out)))
+ ok("batch/predict_input_order", [entry["answers"].get("triage", {}).get("noul")
+ for entry in out["requests"]] == [None, None, 0.7],
+ repr(out["requests"]))
+ ok("batch/predict_department", out["requests"][0]["answers"]["department"]["choice"] == "billing")
+ ok("batch/predict_routing", out["requests"][1]["routing"]["model"] == "english")
+ ok("batch/predict_device_resident", out["requests"][0].get("device") == "cpu",
+ repr(out["requests"][0].get("device")))
+ ok("batch/predict_counts", out["model_counts"] == {"english": 3}, repr(out["model_counts"]))
+ ok("batch/predict_latency", out["total_latency_ms"] >= 0
+ and abs(out["per_request_latency_ms"] - out["total_latency_ms"] / 3) < 0.01,
+ repr(out["per_request_latency_ms"]))
+
+ # batch_size unset must not be forwarded as None (strict old stubs included).
+ router = BatchRouter()
+ laya_predict_batch(BATCH_REQUESTS, router=router)
+ ok("batch/predict_size_default", router.predict_batch_calls[0][1] is None)
+
+ expect_tool_error("batch/predict_count_mismatch",
+ lambda: laya_predict_batch(BATCH_REQUESTS, router=ShortRouter()),
+ "internal_error")
+
+
+def test_batch_route():
+ expect_tool_error("batch/route_no_router",
+ lambda: laya_route_batch(BATCH_REQUESTS, router=None), "models_not_ready")
+ expect_tool_error("batch/route_no_method",
+ lambda: laya_route_batch(BATCH_REQUESTS, router=FakeRouter()),
+ "internal_error")
+ router = BatchRouter()
+ out = laya_route_batch(BATCH_REQUESTS, router=router)
+ ok("batch/route_one_call", len(router.route_batch_calls) == 1)
+ ok("batch/route_decisions", len(out["decisions"]) == 3
+ and set(out["decisions"][0]) == {"model", "repo", "reason"}, repr(out["decisions"][0]))
+ ok("batch/route_counts", out["model_counts"] == {"english": 3}, repr(out["model_counts"]))
+ # Route-only never predicts.
+ ok("batch/route_no_forward", router.predict_batch_calls == [])
+ expect_tool_error("batch/route_count_mismatch",
+ lambda: laya_route_batch(BATCH_REQUESTS, router=ShortRouter()),
+ "internal_error")
+
+
+# --- decide tool (mocked router, no weights) ----------------------------------
+
+DECIDE_SCHEMA = {
+ "type": "object",
+ "properties": {
+ "department": {"enum": ["billing", "support", "sales"],
+ "description": "Which team should handle this ticket?"},
+ "urgency": {"type": "integer", "minimum": 0, "maximum": 2},
+ "needs_human": {"type": "boolean"},
+ },
+}
+
+
+class TypeEchoRouter(FakeRouter):
+ """FakeRouter that also answers with the real payload shape: the `type`
+ key and a probabilities dict keyed by option/level, like system_one does,
+ so laya.structured's per-field probabilities are exercised."""
+
+ def predict(self, state, questions, **kwargs):
+ out = super().predict(state, questions, **kwargs)
+ for name, spec in questions.items():
+ answer = out["answers"][name]
+ answer["type"] = spec["type"]
+ if spec["type"] == "score":
+ answer["probabilities"] = {"0": 0.1, "1": 0.2, "2": 0.7}
+ return out
+
+
+def test_decide():
+ import laya
+
+ router = TypeEchoRouter()
+ out = laya_decide(STATE, DECIDE_SCHEMA, router=router)
+ ok("decide/keys", set(out) >= {"values", "confidence", "probabilities", "routing", "latency_ms"},
+ repr(sorted(out)))
+ ok("decide/values_choice", out["values"]["department"] == "billing", repr(out["values"]))
+ ok("decide/values_score_level", out["values"]["urgency"] == 2, repr(out["values"]))
+ ok("decide/values_bool", out["values"]["needs_human"] is True, repr(out["values"]))
+ ok("decide/confidence", set(out["confidence"]) == {"department", "urgency", "needs_human"}
+ and out["confidence"]["department"] == 0.94, repr(out["confidence"]))
+ ok("decide/probs_noul_pair", set(out["probabilities"]["needs_human"]) == {"false", "true"}
+ and out["probabilities"]["needs_human"]["true"] == 0.892,
+ repr(out["probabilities"]["needs_human"]))
+ ok("decide/probs_score_offset", set(out["probabilities"]["urgency"]) >= {"0", "1", "2"},
+ repr(out["probabilities"]["urgency"]))
+ ok("decide/score_level_from_probs", out["values"]["urgency"] == 2, repr(out["values"]))
+ ok("decide/device", out.get("device") == "cpu", repr(out.get("device")))
+ ok("decide/routing", out["routing"]["model"] == "english")
+
+ # Same values the core decide() returns for the same runner: the tool is a
+ # thin MCP surface over the documented schema projection, not a second one.
+ core = laya.decide(router, STATE, schema=DECIDE_SCHEMA)
+ ok("decide/parity_with_core", out["values"] == core, "tool=%r core=%r" % (out["values"], core))
+
+ # Score projection follows the argmax of the distribution, offset by minimum.
+ class LowUrgencyRouter(TypeEchoRouter):
+ def predict(self, state, questions, **kwargs):
+ out = super().predict(state, questions, **kwargs)
+ out["answers"]["urgency"] = {"type": "score", "score": 2.0, "confidence": 0.6,
+ "probabilities": {"0": 0.6, "1": 0.3, "2": 0.1}}
+ return out
+
+ out = laya_decide(STATE, DECIDE_SCHEMA, router=LowUrgencyRouter())
+ ok("decide/score_argmax", out["values"]["urgency"] == 0, repr(out["values"]["urgency"]))
+
+ expect_tool_error("decide/bad_state", lambda: laya_decide({}, DECIDE_SCHEMA, router=router),
+ "invalid_state")
+ for bad, why in (({"type": "object"}, "no properties"),
+ ({"type": "object", "properties": {"x": {"type": "string"}}}, "free string"),
+ ({"type": "object", "properties": {"x": {"type": "array"}}}, "array"),
+ ({}, "empty"), ([], "not an object")):
+ expect_tool_error("decide/bad_schema_%s" % why,
+ lambda b=bad: laya_decide(STATE, b, router=router), "invalid_schema")
+ expect_tool_error("decide/bad_model",
+ lambda: laya_decide(STATE, DECIDE_SCHEMA, model="gpt4", router=router),
+ "invalid_model")
+ expect_tool_error("decide/no_router",
+ lambda: laya_decide(STATE, DECIDE_SCHEMA), "models_not_ready")
+ # Explicit model with a direct agent: answered by the agent, no Router needed.
+ # A bare Agent payload carries no routing, which is when the tool synthesises
+ # the "explicit model" routing entry (laya_predict does the same).
+ class BareAgent:
+ device = "cpu"
+
+ def predict(self, state, questions, **kwargs):
+ answers = {}
+ for name, spec in questions.items():
+ if spec["type"] == "choice":
+ answers[name] = {"type": "choice", "choice": "billing", "confidence": 0.9,
+ "probabilities": {"billing": 0.9, "support": 0.1}}
+ elif spec["type"] == "score":
+ answers[name] = {"type": "score", "score": 1.0, "confidence": 0.6,
+ "probabilities": {"0": 0.2, "1": 0.6, "2": 0.2}}
+ else:
+ answers[name] = {"type": "noul", "noul": 0.2, "confidence": 0.8}
+ return {"answers": answers}
+
+ out = laya_decide(STATE, DECIDE_SCHEMA, model="english", agent=BareAgent())
+ ok("decide/direct_agent", out["values"]["department"] == "billing"
+ and out["routing"] == {"model": "english", "repo": None, "reason": "explicit model"},
+ repr(out["routing"]))
+ ok("decide/direct_agent_device", out.get("device") == "cpu", repr(out.get("device")))
+
+
+def test_shortlist_lang_parity():
+ """Auto mode must forward the routed language, so per-language temperatures
+ behave as they do in laya_predict. Passing an explicit model= makes
+ Router.predict re-route with that model, and an explicit-model decision
+ carries no detection -- so the language is lost unless shortlist passes it on.
+ """
+ class DetectingRouter(ShortlistRouter):
+ def route(self, state, questions):
+ return {
+ "model": "multilingual",
+ "repo": "fake/repo",
+ "reason": "Latin script but language looks like 'de'",
+ "detection": {"language": "de", "script": "Latin"},
+ }
+
+ router = DetectingRouter({"multilingual": ShortlistAgent()}, routed="multilingual")
+ out = laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, router=router, embed_fn=_tie_embed)
+ ok("shortlist_lang/auto_forwards_lang", (router.seen_kwargs or {}).get("lang") == "de",
+ repr(router.seen_kwargs))
+ ok("shortlist_lang/auto_still_explicit_model",
+ (router.seen_kwargs or {}).get("model") == "multilingual", repr(router.seen_kwargs))
+ ok("shortlist_lang/auto_routes_once",
+ out["routing"]["model"] == "multilingual", repr(out["routing"]))
+
+ # A decision without a usable language must not invent one.
+ plain = ShortlistRouter({"english": ShortlistAgent()}, routed="english")
+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, router=plain, embed_fn=_tie_embed)
+ ok("shortlist_lang/no_detection_adds_nothing", plain.seen_kwargs == {"model": "english"},
+ repr(plain.seen_kwargs))
+
+ # Explicit-model mode is unchanged: no routing, so no language to forward.
+ agent = ShortlistDirectAgent()
+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="multilingual", agent=agent,
+ embed_fn=_tie_embed)
+ ok("shortlist_lang/explicit_model_untouched", agent.seen is not None)
+
+
+def test_shortlist_embed_cache():
+ """A fixed option list must not be re-embedded on every call.
+
+ Uses the answering checkpoint's own encoder (no injected embed_fn) through a
+ weight-free fake tokenizer/encoder, so the number of tokenizer batches is a
+ direct count of embedding work.
+ """
+ import torch
+
+ class _Out:
+ def __init__(self, hidden):
+ self.last_hidden_state = hidden
+
+ class CountingEncoder:
+ def __init__(self):
+ self.calls = 0
+
+ def __call__(self, input_ids=None, attention_mask=None):
+ self.calls += 1
+ ids = input_ids.float()
+ return _Out(torch.stack([ids, torch.ones_like(ids)], dim=-1))
+
+ class CacheTok:
+ def __init__(self):
+ self.batches = []
+
+ def __call__(self, texts, padding=True, truncation=True, max_length=256, return_tensors="pt"):
+ self.batches.append(list(texts))
+ rows = [[(ord(ch) % 5) + 1 for ch in t][:max_length] or [1] for t in texts]
+ width = max(len(r) for r in rows)
+ ids, mask = [], []
+ for row in rows:
+ pad = width - len(row)
+ ids.append(row + [0] * pad)
+ mask.append([1] * len(row) + [0] * pad)
+ return {"input_ids": torch.tensor(ids, dtype=torch.long),
+ "attention_mask": torch.tensor(mask, dtype=torch.long)}
+
+ def make_agent():
+ enc = CountingEncoder()
+ tok = CacheTok()
+ agent = ShortlistAgent()
+ agent.tok = tok
+ agent.encoder = enc
+ agent.model = type("M", (), {"encoder": enc})()
+ agent.device = torch.device("cpu")
+ return agent
+
+ # Same checkpoint, same option list: the second call re-embeds only the query.
+ # Count embedded texts, not tokenizer batches: the whole set fits in one batch
+ # either way, so batch count cannot tell the two cases apart.
+ agent = make_agent()
+ router = ShortlistRouter({"english": agent})
+
+ def texts_since(mark):
+ return sum(len(b) for b in agent.tok.batches[mark:])
+
+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="english", router=router)
+ first_texts = texts_since(0)
+ ok("shortlist_cache/first_call_embeds_query_and_options", first_texts == 6,
+ "texts=%d batches=%r" % (first_texts, agent.tok.batches))
+
+ # An identical repeat call embeds nothing at all: the query text is the same
+ # too, and cached_embed_fn matches on exact strings.
+ mark = len(agent.tok.batches)
+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="english", router=router)
+ ok("shortlist_cache/identical_repeat_embeds_nothing", texts_since(mark) == 0,
+ "texts=%d batches=%r" % (texts_since(mark), agent.tok.batches[mark:]))
+
+ # A different state is a new query, so a repeat with that state embeds the
+ # query alone and reuses the five cached option rows.
+ mark = len(agent.tok.batches)
+ laya_shortlist({"text": "a completely different customer message"},
+ SHORTLIST_QUESTIONS, k=2, model="english", router=router)
+ ok("shortlist_cache/new_state_still_embedded", texts_since(mark) == 1,
+ "texts=%d batches=%r" % (texts_since(mark), agent.tok.batches[mark:]))
+
+ # A different answering checkpoint must not share another one's embeddings.
+ other = make_agent()
+ other_router = ShortlistRouter({"multilingual": other})
+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="multilingual", router=other_router)
+ ok("shortlist_cache/other_checkpoint_embeds_its_own",
+ sum(len(b) for b in other.tok.batches) == 6,
+ "texts=%d" % sum(len(b) for b in other.tok.batches))
+ ok("shortlist_cache/other_checkpoint_encoder_used", other.encoder.calls == 1,
+ "calls=%d" % other.encoder.calls)
+
+ # Replacing the agent (a reload) starts a fresh cache rather than reusing rows.
+ reloaded = make_agent()
+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="english",
+ router=ShortlistRouter({"english": reloaded}))
+ ok("shortlist_cache/reloaded_agent_re_embeds",
+ sum(len(b) for b in reloaded.tok.batches) == 6,
+ "texts=%d" % sum(len(b) for b in reloaded.tok.batches))
+ ok("shortlist_cache/other_checkpoint_still_cached",
+ other.encoder.calls == 1, "calls=%d" % other.encoder.calls)
+
+ # An injected embed_fn stays the caller's own, uncached.
+ calls = []
+
+ def counting_embed(texts):
+ calls.append(list(texts))
+ return [[0.0, 0.0] for _ in texts]
+
+ inj_router = ShortlistRouter({"english": make_agent()})
+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="english", router=inj_router,
+ embed_fn=counting_embed)
+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="english", router=inj_router,
+ embed_fn=counting_embed)
+ ok("shortlist_cache/injected_embed_not_wrapped", len(calls) == 2,
+ "calls=%d" % len(calls))
+
+
+# --- per-call controls: task / lang / max_len / head_max_len -------------------
+
+class ControlRouter:
+ """A router that records which keywords the tools handed to core.
+
+ `route` and `predict` both take **kwargs on purpose: what these checks read is which keywords
+ arrive, and an absent one has to be distinguishable from a `None` one -- which is exactly what
+ `_overrides` promises.
+ """
+
+ def __init__(self, routed="english"):
+ self._agents = {routed: ShortlistAgent()}
+ self.routed = routed
+ self.predict_calls = []
+ self.predict_questions = []
+ self.route_calls = []
+
+ def route(self, state, questions, **kwargs):
+ self.route_calls.append(kwargs)
+ return {"model": self.routed, "repo": "fake/repo",
+ "reason": "unit-test route %s" % sorted(kwargs)}
+
+ def predict(self, state, questions, **kwargs):
+ self.predict_calls.append(kwargs)
+ self.predict_questions.append(questions)
+ return {"answers": {"department": {"choice": "billing", "confidence": 0.94}},
+ "routing": {"model": kwargs.get("model", self.routed), "repo": "fake/repo",
+ "reason": "unit-test predict %s" % sorted(kwargs)}}
+
+ def load(self, name):
+ return ShortlistAgent()
+
+
+class ControlAgent:
+ """An ``agent=`` injection: it answers, and it remembers being asked."""
+
+ device = "cpu"
+
+ def __init__(self):
+ self.calls = []
+
+ def predict(self, state, questions, **kwargs):
+ self.calls.append(kwargs)
+ return {"answers": {"department": {"choice": "billing", "confidence": 0.94}}}
+
+
+def test_controls_validation():
+ """Each control is checked the way core checks it, and refused in this layer's words."""
+ from laya.router import DEFAULT_MODELS, _ALIASES
+
+ ok("task/none_is_unset", validate_task(None) is None)
+ for name in sorted(DEFAULT_MODELS):
+ ok("task/canonical_%s" % name, validate_task(name) == name)
+ for alias in sorted(_ALIASES):
+ # Accepted because core accepts it, and forwarded as the caller wrote it: `Router._route`
+ # normalises the task itself, and its `reason` quotes the spelling that was asked for, so
+ # canonicalising here would rewrite the explanation the caller reads back.
+ ok("task/alias_%s" % alias, validate_task(alias) == alias, repr(validate_task(alias)))
+ # What the sweep really protects: every name core's registry holds is accepted here, so this
+ # layer can never be narrower than `router.predict(task=...)` the way a copied list was.
+ from laya.router import normalise_name
+
+ for alias in sorted(_ALIASES):
+ ok("task/resolves_%s" % alias, normalise_name(validate_task(alias)) == _ALIASES[alias],
+ "%s -> %s" % (alias, normalise_name(validate_task(alias))))
+ # The underscore form is what the CLI and the question ids use, and `Router._route` remaps it
+ # before it looks anything up. The tool remaps the same way but forwards the caller's own
+ # spelling, so `routing.reason` still reads as the request that was made.
+ for spelling in ("typed_decisions", "TYPED_DECISIONS", " typed_decisions ", "typed-decisions"):
+ got = validate_task(spelling)
+ ok("task/workflow_%r" % spelling, got == spelling, repr(got))
+ for bad in ("nope", "", " ", "english-ish", 5, [], {}, True):
+ expect_tool_error("task/rejected_%r" % (bad,), lambda b=bad: validate_task(b), "invalid_task")
+ try:
+ validate_task("nope")
+ message = ""
+ except ToolError as exc:
+ message = exc.message
+ ok("task/error_lists_checkpoints", all(n in message for n in sorted(DEFAULT_MODELS)),
+ repr(message))
+
+ ok("lang/none_is_unset", validate_lang(None) is None)
+ # Verbatim, including the codes that mean nothing to core: a blank lang falls through to
+ # detection rather than failing, and whether a code names a language is core's question.
+ for code in ("de", "en-US", "pt", "en_US.UTF-8", "", " ", "DE"):
+ ok("lang/verbatim_%r" % code, validate_lang(code) == code)
+ for bad in (5, ["de"], {"lang": "de"}, True):
+ expect_tool_error("lang/rejected_%r" % (bad,), lambda b=bad: validate_lang(b), "invalid_lang")
+
+ ok("budget/none_is_unset", validate_budget(None, "max_len") is None)
+ for value in (1, 8, 512, 8192):
+ ok("budget/ok_%d" % value, validate_budget(value, "max_len") == value)
+ # The code carries the argument's own name, so a client that set one wrongly learns which.
+ for bad in (0, -1, 1.5, 384.0, "384", True, [], {}, " "):
+ for arg in ("max_len", "head_max_len"):
+ expect_tool_error("budget/rejected_%s_%r" % (arg, bad),
+ lambda b=bad, a=arg: validate_budget(b, a), "invalid_%s" % arg)
+ # 384.0 is a float and core slices with it (`ids[:max_len]`), which is a TypeError in the middle
+ # of a forward pass. JSON writes both 384 and 384.0, so this is a real input, not a corner.
+ expect_tool_error("budget/float_is_rejected", lambda: validate_budget(384.0, "max_len"),
+ "invalid_max_len")
+
+ ok("overrides/all_unset", _overrides(None, None, None, None) == {})
+ ok("overrides/drops_unset",
+ _overrides(None, "de", None, 384) == {"lang": "de", "head_max_len": 384},
+ repr(_overrides(None, "de", None, 384)))
+ ok("overrides/keeps_falsy_lang", _overrides(None, "", None, None) == {"lang": ""},
+ repr(_overrides(None, "", None, None)))
+ ok("overrides/names_in_order",
+ list(_overrides("t", "l", 1, 2)) == ["task", "lang", "max_len", "head_max_len"])
+
+
+def test_controls_predict():
+ """What `laya_predict` forwards, and what it refuses to forward."""
+ # The baseline these checks protect: a call that sets no control reaches core exactly as it did
+ # before the controls existed -- no keywords at all.
+ router = ControlRouter()
+ laya_predict(STATE, QUESTIONS, router=router)
+ ok("predict/no_controls_call", router.predict_calls == [{}], repr(router.predict_calls))
+ laya_predict(STATE, QUESTIONS, model="english", router=router)
+ ok("predict/pinned_model_only_kwarg", router.predict_calls[-1] == {"model": "english"},
+ repr(router.predict_calls[-1]))
+
+ router = ControlRouter()
+ out = laya_predict(STATE, QUESTIONS, task="typed_decisions", lang="de",
+ max_len=1024, head_max_len=512, router=router)
+ ok("predict/all_controls_forwarded",
+ router.predict_calls == [{"task": "typed_decisions", "lang": "de",
+ "max_len": 1024, "head_max_len": 512}],
+ repr(router.predict_calls))
+ ok("predict/answers_untouched", out["answers"]["department"]["choice"] == "billing")
+
+ router = ControlRouter()
+ laya_predict(STATE, QUESTIONS, lang="de", router=router)
+ ok("predict/lang_only", router.predict_calls == [{"lang": "de"}], repr(router.predict_calls))
+ router = ControlRouter()
+ laya_predict(STATE, QUESTIONS, head_max_len=384, router=router)
+ ok("predict/budget_only", router.predict_calls == [{"head_max_len": 384}],
+ repr(router.predict_calls))
+
+ # A pinned checkpoint plus a task is two answers to one question: `Router._route` checks model
+ # first and never reads the task, and `Agent.system_one` does not accept it at all.
+ for model in ("english", "laya", "ML"):
+ router = ControlRouter()
+ expect_tool_error("predict/model_plus_task_%s" % model,
+ lambda m=model: laya_predict(STATE, QUESTIONS, model=m,
+ task="typed_decisions", router=router),
+ "invalid_task")
+ ok("predict/model_plus_task_no_call_%s" % model, router.predict_calls == [],
+ repr(router.predict_calls))
+
+ # Everything is checked before core is called, so a bad value costs no forward pass.
+ router = ControlRouter()
+ expect_tool_error("predict/bad_budget_before_core",
+ lambda: laya_predict(STATE, QUESTIONS, max_len=0, router=router),
+ "invalid_max_len")
+ ok("predict/bad_budget_no_call", router.predict_calls == [], repr(router.predict_calls))
+ router = ControlRouter()
+ expect_tool_error("predict/bad_task_before_core",
+ lambda: laya_predict(STATE, QUESTIONS, task="nope", router=router),
+ "invalid_task")
+ ok("predict/bad_task_no_call", router.predict_calls == [], repr(router.predict_calls))
+
+ # The agent branch: budget and lang reach a checkpoint directly, task cannot mean anything.
+ agent = ControlAgent()
+ laya_predict(STATE, QUESTIONS, model="english", agent=agent)
+ ok("predict/agent_no_controls_call", agent.calls == [{}], repr(agent.calls))
+ agent = ControlAgent()
+ out = laya_predict(STATE, QUESTIONS, model="english", lang="de", max_len=1024,
+ head_max_len=512, agent=agent)
+ ok("predict/agent_budget_forwarded",
+ agent.calls == [{"lang": "de", "max_len": 1024, "head_max_len": 512}], repr(agent.calls))
+ ok("predict/agent_routing_recorded", out["routing"]["model"] == "english", repr(out["routing"]))
+ agent = ControlAgent()
+ expect_tool_error("predict/agent_task_refused",
+ lambda a=agent: laya_predict(STATE, QUESTIONS, model="english",
+ task="typed_decisions", agent=a),
+ "invalid_task")
+ ok("predict/agent_task_no_call", agent.calls == [], repr(agent.calls))
+
+ # A router whose predict() accepts only state and questions still answers a plain call.
+ class LegacyRouter:
+ _agents = {"english": FakeAgent()}
+
+ def predict(self, state, questions):
+ return {"answers": {"department": {"choice": "other", "confidence": 0.6}}}
+
+ out = laya_predict(STATE, QUESTIONS, router=LegacyRouter())
+ ok("predict/legacy_router_untouched", out["answers"]["department"]["choice"] == "other")
+
+
+def test_controls_route():
+ """`laya_route` takes the routing overrides, so a route can explain a pinned predict."""
+ router = ControlRouter(routed="multilingual")
+ laya_route(STATE, QUESTIONS, router=router)
+ ok("route/no_overrides_call", router.route_calls == [{}], repr(router.route_calls))
+
+ router = ControlRouter(routed="multilingual")
+ out = laya_route(STATE, QUESTIONS, task="typed_decisions", lang="de", router=router)
+ ok("route/overrides_forwarded",
+ router.route_calls == [{"task": "typed_decisions", "lang": "de"}], repr(router.route_calls))
+ ok("route/decision_shape", set(out) == {"model", "repo", "reason"}, repr(out))
+
+ # `auto` is this layer's sentinel and core has no such name: it means "do not pin", so it is
+ # dropped rather than forwarded as a checkpoint that does not exist.
+ for unset in (None, "auto", "AUTO", " auto "):
+ router = ControlRouter()
+ laya_route(STATE, QUESTIONS, model=unset, router=router)
+ ok("route/auto_is_absent_%r" % (unset,), router.route_calls == [{}], repr(router.route_calls))
+ router = ControlRouter()
+ laya_route(STATE, QUESTIONS, model="en", router=router)
+ ok("route/model_canonical", router.route_calls == [{"model": "english"}],
+ repr(router.route_calls))
+ router = ControlRouter()
+ expect_tool_error("route/model_plus_task",
+ lambda: laya_route(STATE, QUESTIONS, model="english",
+ task="typed_decisions", router=router),
+ "invalid_task")
+ ok("route/refused_before_core", router.route_calls == [], repr(router.route_calls))
+ for bad in ("nope", 5, []):
+ router = ControlRouter()
+ expect_tool_error("route/bad_task_%r" % (bad,),
+ lambda b=bad: laya_route(STATE, QUESTIONS, task=b, router=router),
+ "invalid_task")
+ ok("route/bad_task_no_call_%r" % (bad,), router.route_calls == [], repr(router.route_calls))
+ router = ControlRouter()
+ expect_tool_error("route/bad_lang",
+ lambda: laya_route(STATE, QUESTIONS, lang=["de"], router=router),
+ "invalid_lang")
+ ok("route/bad_lang_no_call", router.route_calls == [], repr(router.route_calls))
+ # No budget on a route: there is no forward pass to size, so the parameter does not exist.
+ try:
+ laya_route(STATE, QUESTIONS, max_len=512, router=ControlRouter())
+ ok("route/no_budget_param", False, "accepted max_len")
+ except TypeError:
+ ok("route/no_budget_param", True)
+
+
+def test_controls_shortlist():
+ """The budget reaches the answering pass; the task reaches only the route that chose it."""
+ small = {"dept": {"type": "choice", "instructions": "pick",
+ "criteria": {"a": "A", "b": "B", "c": "C", "d": "D", "e": "E"}}}
+
+ router = ControlRouter(routed="multilingual")
+ laya_shortlist(STATE, small, k=2, router=router, embed_fn=_tie_embed)
+ ok("shortlist/no_controls_route", router.route_calls == [{}], repr(router.route_calls))
+ ok("shortlist/no_controls_predict", router.predict_calls == [{"model": "multilingual"}],
+ repr(router.predict_calls))
+
+ router = ControlRouter(routed="multilingual")
+ out = laya_shortlist(STATE, small, k=2, task="typed_decisions", lang="de",
+ max_len=1024, head_max_len=384, router=router, embed_fn=_tie_embed)
+ ok("shortlist/route_sees_task",
+ router.route_calls == [{"task": "typed_decisions", "lang": "de"}], repr(router.route_calls))
+ ok("shortlist/predict_sees_budget",
+ router.predict_calls == [{"model": "multilingual", "lang": "de",
+ "max_len": 1024, "head_max_len": 384}],
+ repr(router.predict_calls))
+ ok("shortlist/shortlist_still_ran", out["shortlist"]["dept"]["k"] == 2
+ and len(out["shortlist"]["dept"]["labels"]) == 2, repr(out["shortlist"]["dept"]))
+
+ router = ControlRouter(routed="multilingual")
+ laya_shortlist(STATE, small, k=2, model="english", lang="de", head_max_len=384,
+ router=router, embed_fn=_tie_embed)
+ ok("shortlist/pinned_skips_route", router.route_calls == [], repr(router.route_calls))
+ ok("shortlist/pinned_budget",
+ router.predict_calls == [{"model": "english", "lang": "de", "head_max_len": 384}],
+ repr(router.predict_calls))
+
+ router = ControlRouter()
+ expect_tool_error("shortlist/pinned_plus_task",
+ lambda: laya_shortlist(STATE, small, k=2, model="english",
+ task="typed_decisions", router=router,
+ embed_fn=_tie_embed),
+ "invalid_task")
+ ok("shortlist/pinned_plus_task_no_call", router.predict_calls == [], repr(router.predict_calls))
+
+ agent = ControlAgent()
+ laya_shortlist(STATE, small, k=2, model="english", agent=agent, lang="de",
+ head_max_len=384, embed_fn=_tie_embed)
+ ok("shortlist/agent_budget", agent.calls == [{"lang": "de", "head_max_len": 384}],
+ repr(agent.calls))
+ agent = ControlAgent()
+ expect_tool_error("shortlist/agent_task_refused",
+ lambda a=agent: laya_shortlist(STATE, small, k=2, model="english",
+ task="typed", agent=a, embed_fn=_tie_embed),
+ "invalid_task")
+ ok("shortlist/agent_task_no_call", agent.calls == [], repr(agent.calls))
+
+ # A bad budget is refused before the embedding pass, which is the expensive half.
+ calls = []
+
+ def counting_embed(texts):
+ calls.append(len(texts))
+ return _tie_embed(texts)
+
+ router = ControlRouter()
+ expect_tool_error("shortlist/bad_budget_before_embedding",
+ lambda: laya_shortlist(STATE, small, k=2, head_max_len="384", router=router,
+ embed_fn=counting_embed),
+ "invalid_head_max_len")
+ ok("shortlist/bad_budget_no_embedding", calls == [], repr(calls))
+
+
+def test_controls_preset():
+ """The preset fixes the questions, not the route or the budget."""
+ def builder(attr):
+ return {"probe": {"type": "noul", "instructions": "Does the `body` need a human?"}}
+
+ router = ControlRouter()
+ laya_preset("guard", STATE, router=router, preset_builder=builder)
+ ok("preset/no_controls_call", router.predict_calls == [{}], repr(router.predict_calls))
+
+ router = ControlRouter()
+ out = laya_preset("guard", STATE, task="typed_decisions", lang="de", max_len=1024,
+ head_max_len=384, router=router, preset_builder=builder)
+ ok("preset/controls_forwarded",
+ router.predict_calls == [{"task": "typed_decisions", "lang": "de",
+ "max_len": 1024, "head_max_len": 384}],
+ repr(router.predict_calls))
+ # The questions the preset builder produced are what core was asked, alongside the controls --
+ # the preset supplies the questions, the caller supplies how they get answered.
+ ok("preset/questions_reached_core",
+ list(router.predict_questions[-1]) == ["probe"], repr(router.predict_questions[-1]))
+ ok("preset/result_shape", set(out) >= {"answers", "routing", "latency_ms"}, repr(sorted(out)))
+
+ for args, code in (({"max_len": -1}, "invalid_max_len"),
+ ({"task": "nope"}, "invalid_task"),
+ ({"lang": 7}, "invalid_lang"),
+ ({"head_max_len": "512"}, "invalid_head_max_len")):
+ router = ControlRouter()
+ expect_tool_error("preset/rejected_%r" % (args,),
+ lambda a=args: laya_preset("guard", STATE, router=router,
+ preset_builder=builder, **a),
+ code)
+ ok("preset/rejected_no_call_%r" % (args,), router.predict_calls == [],
+ repr(router.predict_calls))
+
+
+def test_controls_signature_and_schema():
+ """The keywords a client can see, and that none of them became required.
+
+ A new required argument would break every prompt already in the wild; a Python keyword with no
+ schema entry would be a control nobody can send. Both directions are checked.
+ """
+ import inspect
+
+ controls = ["task", "lang", "max_len", "head_max_len"]
+ for fn, want in ((laya_predict, controls), (laya_shortlist, controls),
+ (laya_preset, controls), (laya_route, ["model", "task", "lang"])):
+ params = inspect.signature(fn).parameters
+ for name in want:
+ ok("signature/%s_has_%s" % (fn.__name__, name), name in params, repr(sorted(params)))
+ ok("signature/%s_%s_default_none" % (fn.__name__, name),
+ params[name].default is None, repr(params[name].default))
+ ok("signature/%s_%s_keyword_only" % (fn.__name__, name),
+ params[name].kind is inspect.Parameter.KEYWORD_ONLY)
+ ok("signature/predict_positionals_kept",
+ list(inspect.signature(laya_predict).parameters)[:3] == ["state", "questions", "model"],
+ repr(list(inspect.signature(laya_predict).parameters)))
+
+ by_name = {t.name: t for t in asyncio.run(mcp_server.list_tools())}
+ for name, want in (("laya_predict", controls), ("laya_shortlist", controls),
+ ("laya_preset", controls), ("laya_route", ["model", "task", "lang"])):
+ tool = by_name[name]
+ schema = tool.input_schema if hasattr(tool, "input_schema") else tool.inputSchema
+ props = schema.get("properties", {})
+ required = schema.get("required", [])
+ for arg in want:
+ ok("schema/%s_exposes_%s" % (name, arg), arg in props, repr(sorted(props)))
+ spec = props.get(arg, {})
+ ok("schema/%s_%s_nullable" % (name, arg),
+ {"type": "null"} in (spec.get("anyOf") or []), repr(spec))
+ ok("schema/%s_%s_default_none" % (name, arg),
+ spec.get("default", "missing") is None, repr(spec))
+ ok("schema/%s_%s_not_required" % (name, arg), arg not in required, repr(required))
+ ok("schema/%s_desc_documents_%s" % (name, arg), arg in tool.description.lower())
+ ok("schema/%s_required_unchanged" % name,
+ required == (["preset", "state"] if name == "laya_preset" else ["state", "questions"]),
+ repr(required))
+ def test_question_validation_matches_the_agent():
+ """MCP must reject a bad question the same way the agent does, and say so.
+
+ `Agent._check_question` is the contract: it runs before encoding and raises `ValueError`
+ for a caller mistake. The MCP tools used to check only part of it, so the rest reached the
+ agent, and `_wrap` mapped the `ValueError` to `internal_error` -- the code this layer
+ reserves for a tool that broke ("predict returned non-object", "router has no route()").
+ An MCP client was told the server failed when it had sent a malformed question.
+
+ This is the same shape as the serve-side test that calls the real guard from a stub
+ router: no weights, and the core guard is the oracle rather than a second hand-written
+ copy of the rules.
+ """
+ from laya.agent import Agent
+
+ def core_rejects(qid, qdef):
+ try:
+ Agent._check_question(qid, dict(qdef))
+ except ValueError:
+ return True
+ except Exception: # noqa: BLE001 -- any refusal is still a refusal
+ return True
+ return False
+
+ # (label, question) for every rule the agent enforces and the tools did not.
+ parity = [
+ ("noul_criteria_bad_key",
+ {"type": "noul", "instructions": "is true?",
+ "criteria": {"yes": "affirmative", "no": "negative"}}),
+ ("noul_criteria_extra_key",
+ {"type": "noul", "instructions": "is true?",
+ "criteria": {"true": "yes", "false": "no", "maybe": "perhaps"}}),
+ ("noul_labels_wrong_names",
+ {"type": "noul", "instructions": "is true?", "labels": {"A": "yes", "B": "no"}}),
+ ("noul_labels_missing_true",
+ {"type": "noul", "instructions": "is true?", "labels": {"false": "no"}}),
+ ("noul_labels_identical",
+ {"type": "noul", "instructions": "is true?",
+ "labels": {"false": "same", "true": "same"}}),
+ ("noul_labels_empty",
+ {"type": "noul", "instructions": "is true?", "labels": {"false": "", "true": "yes"}}),
+ ("noul_labels_not_object",
+ {"type": "noul", "instructions": "is true?", "labels": ["yes", "no"]}),
+ # A label is text, not a value. The agent checks the type before using it, so these
+ # have to be refused here too -- stringifying them would accept exactly what the agent
+ # rejects, and the failure would surface as an internal_error instead.
+ ("noul_labels_numeric",
+ {"type": "noul", "instructions": "is true?", "labels": {"false": 0, "true": 1}}),
+ ("noul_labels_mixed_types",
+ {"type": "noul", "instructions": "is true?", "labels": {"false": "no", "true": 1}}),
+ ("noul_labels_bool",
+ {"type": "noul", "instructions": "is true?", "labels": {"false": False, "true": True}}),
+ ("noul_labels_null",
+ {"type": "noul", "instructions": "is true?", "labels": {"false": None, "true": "yes"}}),
+ ("noul_labels_list",
+ {"type": "noul", "instructions": "is true?", "labels": {"false": ["no"], "true": "yes"}}),
+ ("noul_labels_whitespace_equal",
+ {"type": "noul", "instructions": "is true?", "labels": {"false": " same ", "true": "same"}}),
+ ("score_level_null",
+ {"type": "score", "instructions": "how bad", "criteria": ["fine", None]}),
+ ("score_level_null_first",
+ {"type": "score", "instructions": "how bad", "criteria": [None, "fine"]}),
+ ("labels_on_choice",
+ {"type": "choice", "instructions": "which?", "criteria": {"a": "first"},
+ "labels": {"false": "no", "true": "yes"}}),
+ ]
+ for label, qdef in parity:
+ ok("question_parity/core_rejects_%s" % label, core_rejects("q", qdef))
+ expect_tool_error("question_parity/mcp_rejects_%s" % label,
+ lambda d=qdef: validate_questions({"q": d}), "invalid_questions")
+
+ # The other direction is deliberately not asserted: the tools reject a few shapes the
+ # agent tolerates (a choice `criteria` list, a non-string `instructions`). Widening what
+ # MCP accepts is a separate contract decision and is not what this change is about.
+ accepted_core = [
+ ("choice_criteria_list", {"type": "choice", "instructions": "which?",
+ "criteria": ["a", "b"]}),
+ ]
+ for label, qdef in accepted_core:
+ ok("question_parity/core_accepts_%s" % label, not core_rejects("q", qdef))
+
+
+ def test_a_bad_question_is_a_caller_error_not_a_server_fault():
+ """End to end through the wrapper: the client must see `invalid_questions`.
+
+ The wrapper turns a `ToolError` into that code and preserves its message, while any other
+ exception becomes `internal_error`. So the only thing standing between a malformed question
+ and a "the server broke" answer is whether the tools reject it themselves.
+ """
+ import json as _json
+
+ from laya.mcp import server as server_mod
+
+ class ValidatingAgent:
+ """The real guard, no checkpoint: what `Agent.system_one` runs before encoding."""
+
+ def predict(self, state, questions, **kwargs):
+ from laya.agent import Agent
+ for qid, qdef in questions.items():
+ Agent._check_question(qid, qdef)
+ return {"answers": {qid: {"type": "noul", "noul": 0.5, "noul_label": None}
+ for qid in questions}}
+
+ class Router:
+ loaded = ["english"]
+
+ def predict(self, state, questions, model=None, **kwargs):
+ return ValidatingAgent().predict(state, questions)
+
+ def route(self, state, questions, **kwargs):
+ return {"model": "english", "repo": "r", "reason": "stub"}
+
+ for label, qdef in (
+ ("noul_criteria_bad_key", {"type": "noul", "instructions": "is true?",
+ "criteria": {"yes": "affirmative", "no": "negative"}}),
+ ("noul_labels_wrong_names", {"type": "noul", "instructions": "is true?",
+ "labels": {"A": "yes", "B": "no"}}),
+ ("noul_labels_numeric", {"type": "noul", "instructions": "is true?",
+ "labels": {"false": 0, "true": 1}}),
+ ("noul_labels_mixed_types", {"type": "noul", "instructions": "is true?",
+ "labels": {"false": "no", "true": 1}}),
+ ("score_level_null", {"type": "score", "instructions": "how bad",
+ "criteria": ["fine", None]}),
+ ):
+ try:
+ server_mod._wrap(server_mod.laya_predict, state=STATE, questions={"q": qdef},
+ model="english", router=Router())
+ ok("caller_error/rejected_%s" % label, False, "no error raised")
+ except Exception as exc: # noqa: BLE001 -- the wrapper raises McpToolError
+ payload = getattr(exc, "message", None) or str(exc)
+ try:
+ code = _json.loads(payload).get("error")
+ except Exception: # noqa: BLE001
+ code = None
+ ok("caller_error/%s" % label, code == "invalid_questions", "got %r" % (code,))
+
+
def test_timeout_removed():
# The per-call timeout was removed: a ThreadPoolExecutor shutdown waits for
# the work anyway, and MCP clients apply their own request timeout. The tool
@@@ -1584,18 -709,8 +1730,20 @@@ test_model_forwarding(
test_shape()
test_question_forwarding()
test_shortlist()
+test_batch_validation()
+test_batch_predict()
+test_batch_route()
+test_decide()
+test_shortlist_lang_parity()
+test_shortlist_embed_cache()
+test_controls_validation()
+test_controls_predict()
+test_controls_route()
+test_controls_shortlist()
+test_controls_preset()
+test_controls_signature_and_schema()
+ test_question_validation_matches_the_agent()
+ test_a_bad_question_is_a_caller_error_not_a_server_fault()
test_timeout_removed()
test_models_from_env()
test_server_registration()