Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(mcp): reject a malformed question as a caller error, not a server fault

Bruce-Yii0b1ccf781fc6
diff --git a/laya/mcp/tools.py b/laya/mcp/tools.py
index 810f5edb..f6cec666 100644
--- a/laya/mcp/tools.py
+++ b/laya/mcp/tools.py
@@ -31,6 +31,29 @@ VALID_TYPES = {"choice", "score", "noul"}
VALID_MODELS = {"auto", "english", "multilingual", "typed-decisions"}
+def _check_noul_labels(name: str, labels: Any) -> None:
+ """`labels` renames a noul answer's two model-facing texts; mirror the agent's rule.
+
+ `laya.common._resolve_noul_labels` is the contract: exactly `false` and `true`, each a
+ distinct non-empty string. The agent raises `ValueError` on anything else, and the tool
+ wrapper turns a `ValueError` into `internal_error` -- the code this layer uses for a tool
+ that broke. So the check is repeated here rather than imported, both to keep the message
+ in this layer's `questions[...]` voice and to keep `laya.mcp.tools` free of a torch import.
+ """
+ if not isinstance(labels, dict) or set(labels) != {"false", "true"}:
+ raise ToolError(
+ "invalid_questions",
+ f"questions[{name}].labels must map exactly 'false' and 'true' to distinct "
+ f"non-empty strings",
+ )
+ texts = [str(labels[k]).strip() for k in ("false", "true")]
+ if not texts[0] or not texts[1] or texts[0] == texts[1]:
+ raise ToolError(
+ "invalid_questions",
+ f"questions[{name}].labels must give 'false' and 'true' distinct non-empty strings",
+ )
+
+
def validate_questions(questions: Any) -> dict:
if not isinstance(questions, dict) or not questions:
raise ToolError(
@@ -70,6 +93,15 @@ def validate_questions(questions: Any) -> dict:
"invalid_questions",
f"questions[{name}].criteria must be a non-empty list of rubric levels",
)
+ # A null level is a hole in the rubric. The agent rejects it before encoding, so
+ # accepting it here only moved the failure past the point where the tool could
+ # still name the question.
+ if None in criteria:
+ raise ToolError(
+ "invalid_questions",
+ f"questions[{name}].criteria has a null level at index "
+ f"{list(criteria).index(None)}; give every level a description",
+ )
entry["criteria"] = list(criteria)
else: # noul
if criteria is not None:
@@ -78,9 +110,27 @@ def validate_questions(questions: Any) -> dict:
"invalid_questions",
f"questions[{name}].criteria must be an object when present (noul)",
)
+ # `render_options` reads these descriptions by name, so a dict keyed any other
+ # way is not a noul description at all -- the agent rejects it rather than
+ # silently falling back to the default pair.
+ keys = {str(k).lower() for k in criteria}
+ if not keys <= {"true", "false"}:
+ raise ToolError(
+ "invalid_questions",
+ f"questions[{name}].criteria must be keyed only 'true'/'false' "
+ f"(either or both, omitting it is fine), got {sorted(keys)}. To word "
+ f"the answer differently set 'labels' instead.",
+ )
entry["criteria"] = {str(k): v for k, v in criteria.items()}
- if "labels" in spec:
- entry["labels"] = spec["labels"]
+ if "labels" in spec:
+ # noul-only, like the agent: on any other type it is a caller mistake.
+ if qtype != "noul":
+ raise ToolError(
+ "invalid_questions",
+ f"questions[{name}].labels is only supported for noul questions",
+ )
+ _check_noul_labels(name, spec["labels"])
+ entry["labels"] = spec["labels"]
cleaned[name] = entry
return cleaned
diff --git a/tests/test_mcp.py b/tests/test_mcp.py
index 4d3afab5..693e8b73 100644
--- a/tests/test_mcp.py
+++ b/tests/test_mcp.py
@@ -122,8 +122,17 @@ def test_schema():
ok("schema/questions_score_ok", out["urg"]["criteria"] == ["low", "high"])
out = validate_questions({"risk": {"type": "noul", "instructions": "is true?"}})
ok("schema/questions_noul_ok", out["risk"]["type"] == "noul")
- out = validate_questions({"risk": {"type": "noul", "instructions": "is true?", "criteria": {"yes": "affirmative"}}})
+ out = validate_questions({"risk": {"type": "noul", "instructions": "is true?",
+ "criteria": {"true": "affirmative", "false": "negative"}}})
ok("schema/questions_noul_criteria_ok", "criteria" in out["risk"])
+ # A noul question takes its criteria keyed only 'true'/'false'. The agent rejects anything
+ # else with a ValueError, so accepting 'yes'/'no' here only moved the failure deeper, where
+ # the tool wrapper reported a caller mistake as an internal_error.
+ expect_tool_error("schema/questions_noul_criteria_bad_key",
+ lambda: validate_questions(
+ {"risk": {"type": "noul", "instructions": "is true?",
+ "criteria": {"yes": "affirmative", "no": "negative"}}}),
+ "invalid_questions")
bad_questions = [
None,
@@ -505,6 +514,124 @@ def test_shortlist():
and out["routing"]["reason"] == "explicit model", repr(out["routing"]))
+def test_question_validation_matches_the_agent():
+ """MCP must reject a bad question the same way the agent does, and say so.
+
+ `Agent._check_question` is the contract: it runs before encoding and raises `ValueError`
+ for a caller mistake. The MCP tools used to check only part of it, so the rest reached the
+ agent, and `_wrap` mapped the `ValueError` to `internal_error` -- the code this layer
+ reserves for a tool that broke ("predict returned non-object", "router has no route()").
+ An MCP client was told the server failed when it had sent a malformed question.
+
+ This is the same shape as the serve-side test that calls the real guard from a stub
+ router: no weights, and the core guard is the oracle rather than a second hand-written
+ copy of the rules.
+ """
+ from laya.agent import Agent
+
+ def core_rejects(qid, qdef):
+ try:
+ Agent._check_question(qid, dict(qdef))
+ except ValueError:
+ return True
+ except Exception: # noqa: BLE001 -- any refusal is still a refusal
+ return True
+ return False
+
+ # (label, question) for every rule the agent enforces and the tools did not.
+ parity = [
+ ("noul_criteria_bad_key",
+ {"type": "noul", "instructions": "is true?",
+ "criteria": {"yes": "affirmative", "no": "negative"}}),
+ ("noul_criteria_extra_key",
+ {"type": "noul", "instructions": "is true?",
+ "criteria": {"true": "yes", "false": "no", "maybe": "perhaps"}}),
+ ("noul_labels_wrong_names",
+ {"type": "noul", "instructions": "is true?", "labels": {"A": "yes", "B": "no"}}),
+ ("noul_labels_missing_true",
+ {"type": "noul", "instructions": "is true?", "labels": {"false": "no"}}),
+ ("noul_labels_identical",
+ {"type": "noul", "instructions": "is true?",
+ "labels": {"false": "same", "true": "same"}}),
+ ("noul_labels_empty",
+ {"type": "noul", "instructions": "is true?", "labels": {"false": "", "true": "yes"}}),
+ ("noul_labels_not_object",
+ {"type": "noul", "instructions": "is true?", "labels": ["yes", "no"]}),
+ ("score_level_null",
+ {"type": "score", "instructions": "how bad", "criteria": ["fine", None]}),
+ ("score_level_null_first",
+ {"type": "score", "instructions": "how bad", "criteria": [None, "fine"]}),
+ ("labels_on_choice",
+ {"type": "choice", "instructions": "which?", "criteria": {"a": "first"},
+ "labels": {"false": "no", "true": "yes"}}),
+ ]
+ for label, qdef in parity:
+ ok("question_parity/core_rejects_%s" % label, core_rejects("q", qdef))
+ expect_tool_error("question_parity/mcp_rejects_%s" % label,
+ lambda d=qdef: validate_questions({"q": d}), "invalid_questions")
+
+ # The other direction is deliberately not asserted: the tools reject a few shapes the
+ # agent tolerates (a choice `criteria` list, a non-string `instructions`). Widening what
+ # MCP accepts is a separate contract decision and is not what this change is about.
+ accepted_core = [
+ ("choice_criteria_list", {"type": "choice", "instructions": "which?",
+ "criteria": ["a", "b"]}),
+ ]
+ for label, qdef in accepted_core:
+ ok("question_parity/core_accepts_%s" % label, not core_rejects("q", qdef))
+
+
+def test_a_bad_question_is_a_caller_error_not_a_server_fault():
+ """End to end through the wrapper: the client must see `invalid_questions`.
+
+ The wrapper turns a `ToolError` into that code and preserves its message, while any other
+ exception becomes `internal_error`. So the only thing standing between a malformed question
+ and a "the server broke" answer is whether the tools reject it themselves.
+ """
+ import json as _json
+
+ from laya.mcp import server as server_mod
+
+ class ValidatingAgent:
+ """The real guard, no checkpoint: what `Agent.system_one` runs before encoding."""
+
+ def predict(self, state, questions, **kwargs):
+ from laya.agent import Agent
+ for qid, qdef in questions.items():
+ Agent._check_question(qid, qdef)
+ return {"answers": {qid: {"type": "noul", "noul": 0.5, "noul_label": None}
+ for qid in questions}}
+
+ class Router:
+ loaded = ["english"]
+
+ def predict(self, state, questions, model=None, **kwargs):
+ return ValidatingAgent().predict(state, questions)
+
+ def route(self, state, questions, **kwargs):
+ return {"model": "english", "repo": "r", "reason": "stub"}
+
+ for label, qdef in (
+ ("noul_criteria_bad_key", {"type": "noul", "instructions": "is true?",
+ "criteria": {"yes": "affirmative", "no": "negative"}}),
+ ("noul_labels_wrong_names", {"type": "noul", "instructions": "is true?",
+ "labels": {"A": "yes", "B": "no"}}),
+ ("score_level_null", {"type": "score", "instructions": "how bad",
+ "criteria": ["fine", None]}),
+ ):
+ try:
+ server_mod._wrap(server_mod.laya_predict, state=STATE, questions={"q": qdef},
+ model="english", router=Router())
+ ok("caller_error/rejected_%s" % label, False, "no error raised")
+ except Exception as exc: # noqa: BLE001 -- the wrapper raises McpToolError
+ payload = getattr(exc, "message", None) or str(exc)
+ try:
+ code = _json.loads(payload).get("error")
+ except Exception: # noqa: BLE001
+ code = None
+ ok("caller_error/%s" % label, code == "invalid_questions", "got %r" % (code,))
+
+
def test_timeout_removed():
# The per-call timeout was removed: a ThreadPoolExecutor shutdown waits for
# the work anyway, and MCP clients apply their own request timeout. The tool
@@ -563,6 +690,8 @@ test_schema()
test_shape()
test_question_forwarding()
test_shortlist()
+test_question_validation_matches_the_agent()
+test_a_bad_question_is_a_caller_error_not_a_server_fault()
test_timeout_removed()
test_models_from_env()
test_server_registration()