Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(integrations): the LangChain wrappers read nested answers through the shared guard

Aashish610e2780d14b
diff --git a/laya/integrations/langchain.py b/laya/integrations/langchain.py
index 48541bfa..1f4718b4 100644
--- a/laya/integrations/langchain.py
+++ b/laya/integrations/langchain.py
@@ -36,6 +36,8 @@ from ._controls import predict_kwargs as _predict_kwargs
from ._controls import reject_remote_hooks as _reject_remote_hooks
from ._controls import reject_remote_lang_guess as _reject_remote_lang_guess
from ._controls import require_router_controls as _require_router_controls, router_kwargs as _router_kwargs
+from ._guard import answer_number as _answer_number
+from ._guard import answers_map as _answers_map
from ._guard import score_violation_probability as _score_violation_probability
@@ -719,16 +721,26 @@ class LayaGuardrail(_BatchedRunnable, RunnableSerializable):
def _finish(self, result: Dict[str, Any], input: Any) -> Any:
"""Apply the violation scan and the configured action to one decision."""
- answers = result.get("answers", {})
+ # A remote body is arbitrary JSON, so `answers` and each answer are checked before anything
+ # is read out of them -- the same two levels and the same `noul` check `LayaTaskGuard.screen`
+ # and `LayaAgentRouter.screen` already delegate to `_guard`. `result.get("answers", {})` only
+ # defaults when the key is ABSENT: a body whose answer is a string (`{"jailbreak": "nope"}`)
+ # raised AttributeError at `ans.get("type")`, a `noul` of `"high"` or `null` raised TypeError
+ # from `>=`, and a `{"type": "noul"}` with no `noul` at all raised KeyError at `ans["noul"]`
+ # whenever the threshold let the comparison through.
+ where = f"Laya server at {self.base_url}" if self.base_url else "Laya"
+ answers = _answers_map(result, where)
violations: Dict[str, Any] = {}
for qid, ans in answers.items():
t = ans.get("type")
- if t == "noul" and ans.get("noul", 0.0) >= self.threshold:
- violations[qid] = {
- "probability": ans["noul"],
- "confidence": ans.get("confidence", 0.0),
- }
+ if t == "noul":
+ probability = _answer_number(ans, "noul", where)
+ if probability >= self.threshold:
+ violations[qid] = {
+ "probability": probability,
+ "confidence": ans.get("confidence", 0.0),
+ }
elif t == "score":
# `score` is the expected level (0..k-1), not a probability: gate on the
# probability that the level is at or above the middle of the scale.
@@ -900,15 +912,25 @@ class LayaTriage(_BatchedRunnable, RunnableSerializable):
def _finish(self, result: Dict[str, Any], state: Any) -> Dict[str, Any]:
"""Project one triage decision onto the graph state."""
- ans = result.get("answers", {})
+ # Three levels of the same body are read here, and only the first was checked: `answers`
+ # itself (a list raised AttributeError at `.get`), the answer under each question id (a
+ # string raised AttributeError at the nested `.get`), and the `noul` inside it (a `"0.9"` or
+ # `null` raised TypeError from `>=`, a `true` read as 1.0 because bool subclasses int, and a
+ # NaN compared False -- i.e. silently "not urgent"). `_answers_map` validates the first two
+ # and `_answer_number` the third, the way the guardrail screens two lines above do.
+ where = f"Laya server at {self.base_url}" if self.base_url else "Laya"
+ ans = _answers_map(result, where)
+
+ def _noul(qid: str) -> float:
+ return _answer_number(ans.get(qid, {}), "noul", where)
triage_info = {
"intent": ans.get("intent", {}).get("choice"),
"intent_confidence": ans.get("intent", {}).get("confidence"),
- "is_urgent": ans.get("is_urgent", {}).get("noul", 0.0) >= 0.5,
+ "is_urgent": _noul("is_urgent") >= 0.5,
"frustration_score": ans.get("frustration", {}).get("score"),
- "churn_risk": ans.get("churn_risk", {}).get("noul", 0.0) >= 0.5,
- "refund_requested": ans.get("refund_requested", {}).get("noul", 0.0) >= 0.5,
+ "churn_risk": _noul("churn_risk") >= 0.5,
+ "refund_requested": _noul("refund_requested") >= 0.5,
}
if isinstance(state, dict):
diff --git a/tests/test_langchain.py b/tests/test_langchain.py
index 5e493130..61f10f50 100644
--- a/tests/test_langchain.py
+++ b/tests/test_langchain.py
@@ -1942,6 +1942,103 @@ finally:
langchain_module.urllib.request.build_opener = _lc_orig_opener_badans
+# ---------------------------------------------- Nested answer objects and `noul` values
+#
+# The section above stops at `answers` itself. Two consumers -- `LayaGuardrail._finish` and
+# `LayaTriage._finish` -- then trusted everything under it, while `LayaTaskGuard.screen` and
+# `LayaAgentRouter.screen` had already moved to `_guard.answers_map`/`_guard.answer_number` (#1335).
+# Three levels leaked here: an answer that is not an object, a `noul` that is not a number, and a
+# `noul` key the answer does not carry.
+_LC_NESTED_ANSWER = [
+ ({"answers": {"jailbreak": "nope"}}, "answer-str", "guardrail"),
+ ({"answers": {"jailbreak": ["nope"]}}, "answer-list", "guardrail"),
+ ({"answers": {"jailbreak": None}}, "answer-null", "guardrail"),
+ ({"answers": {"jailbreak": {"type": "noul", "noul": "high"}}}, "noul-str", "guardrail"),
+ ({"answers": {"jailbreak": {"type": "noul", "noul": None}}}, "noul-null", "guardrail"),
+ ({"answers": {"jailbreak": {"type": "noul", "noul": True}}}, "noul-bool", "guardrail"),
+ ({"answers": {"jailbreak": {"type": "noul", "noul": float("nan")}}}, "noul-nan", "guardrail"),
+ ({"answers": {"intent": "escalate"}}, "answer-str", "triage"),
+ ({"answers": {"intent": ["escalate"]}}, "answer-list", "triage"),
+ ({"answers": {"intent": None}}, "answer-null", "triage"),
+ ({"answers": {"is_urgent": {"noul": "0.9"}}}, "noul-str", "triage"),
+ ({"answers": {"is_urgent": {"noul": None}}}, "noul-null", "triage"),
+ ({"answers": {"is_urgent": {"noul": True}}}, "noul-bool", "triage"),
+ ({"answers": {"churn_risk": {"noul": float("nan")}}}, "noul-nan", "triage"),
+]
+_lc_orig_opener_nested = langchain_module.urllib.request.build_opener
+try:
+ for _body, _label, _entry in _LC_NESTED_ANSWER:
+ _name = "%s/%s" % (_entry, _label)
+ langchain_module.urllib.request.build_opener = (
+ lambda *a, _r=json.dumps(_body): _RawBodyOpener(_r))
+ _run = (lambda: LayaGuardrail(base_url="http://localhost:8000",
+ api_key="k").invoke("What is the capital of France?")) \
+ if _entry == "guardrail" else \
+ (lambda: LayaTriage(base_url="http://localhost:8000",
+ api_key="k").invoke({"state": "I want a refund"}))
+ try:
+ _run()
+ check_true("nested/%s raises" % _name, False, "no error raised")
+ except RuntimeError as _e:
+ check_true("nested/%s raises" % _name, True)
+ check_true("nested/%s malformed" % _name,
+ "Malformed decision response" in str(_e), str(_e))
+ check_true("nested/%s names the field" % _name,
+ ("'jailbreak'" in str(_e) or "noul" in str(_e) or "'intent'" in str(_e)
+ or "is_urgent" in str(_e) or "churn_risk" in str(_e)), str(_e))
+ except Exception as _e:
+ check_true("nested/%s raises" % _name, False, type(_e).__name__)
+
+ # A `{"type": "noul"}` with no `noul` at all used to raise KeyError: 'noul' as soon as the
+ # threshold let the comparison through, because the value was re-read from the answer after the
+ # `>=` guard. It now reads as the 0.0 the comparison already used, exactly as `LayaTaskGuard`
+ # reports it, so the guardrail answers instead of crashing.
+ langchain_module.urllib.request.build_opener = (
+ lambda *a: _RawBodyOpener(json.dumps({"answers": {"jailbreak": {"type": "noul"}}})))
+ _annotated = LayaGuardrail(base_url="http://localhost:8000", api_key="k",
+ action="annotate", threshold=0.0).invoke(
+ {"state": "What is the capital of France?"})
+ check("nested/guardrail/noul-absent records the default",
+ _annotated["guardrails"]["violations"], {"jailbreak": {"probability": 0.0,
+ "confidence": 0.0}})
+
+ # Controls: a well-formed body projects the same numbers it always did, on both surfaces.
+ langchain_module.urllib.request.build_opener = (
+ lambda *a: _RawBodyOpener(json.dumps({"answers": {
+ "jailbreak": {"type": "noul", "noul": 0.9, "confidence": 0.4},
+ "toxicity": {"type": "score", "score": 3, "probabilities": {"0": 0.0, "1": 0.2,
+ "2": 0.3, "3": 0.5}}}})))
+ _violated = LayaGuardrail(base_url="http://localhost:8000", api_key="k",
+ action="annotate").invoke({"state": "hello"})
+ check("nested/guardrail/well-formed still records noul",
+ _violated["guardrails"]["violations"]["jailbreak"],
+ {"probability": 0.9, "confidence": 0.4})
+ check("nested/guardrail/well-formed still records score",
+ _violated["guardrails"]["violations"]["toxicity"]["probability"], 0.8)
+ langchain_module.urllib.request.build_opener = (
+ lambda *a: _RawBodyOpener(json.dumps({"answers": {
+ "intent": {"type": "choice", "choice": "refund", "confidence": 0.8},
+ "is_urgent": {"type": "noul", "noul": 0.7},
+ "churn_risk": {"type": "noul", "noul": 0.1},
+ "refund_requested": {"type": "noul", "noul": 0.9},
+ "frustration": {"type": "score", "score": 2}}})))
+ check("nested/triage/well-formed still projects",
+ LayaTriage(base_url="http://localhost:8000", api_key="k").invoke(
+ {"state": "I want a refund"})["triage"],
+ {"intent": "refund", "intent_confidence": 0.8, "is_urgent": True,
+ "frustration_score": 2, "churn_risk": False, "refund_requested": True})
+
+ # A missing question in the body is still the no-decision case it was: `{}` answers read as 0.0,
+ # so nothing is refused and nothing is flagged.
+ langchain_module.urllib.request.build_opener = (
+ lambda *a: _RawBodyOpener(json.dumps({"answers": {"intent": {"type": "choice"}}})))
+ check("nested/triage/absent noul questions stay false",
+ LayaTriage(base_url="http://localhost:8000", api_key="k").invoke(
+ {"state": "hi"})["triage"]["is_urgent"], False)
+finally:
+ langchain_module.urllib.request.build_opener = _lc_orig_opener_nested
+
+
# --------------------------------------------------------------- Summary
print(f"PASS: {len(PASS)}")
print(f"FAIL: {len(FAIL)}")