Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(integrations): fail closed on missing or unusable tool decisions and validate confidence threshold
LEVELING2108305ae8a8e8fa
diff --git a/laya/integrations/tools.py b/laya/integrations/tools.pyindex dbd474f5..b8d8d017 100644--- a/laya/integrations/tools.py+++ b/laya/integrations/tools.py@@ -14,6 +14,7 @@ from __future__ import annotationsimport asyncioimport json+import mathimport threadingimport urllib.errorimport urllib.parse@@ -28,13 +29,12 @@ from ._controls import hook_kwargs as _hook_kwargsfrom ._controls import predict_kwargs as _predict_kwargsfrom ._controls import reject_remote_hooks as _reject_remote_hooksfrom ._errors import LayaLowConfidenceError-from ..confidence import _gate_confidence+from ..confidence import _gate_confidence, check_min_confidence-def _gated_confidence(answer: Dict[str, Any]) -> float:- """Extract calibrated confidence score from a Laya decision answer."""- conf = _gate_confidence(answer or {})- return conf if conf is not None else 1.0+def _gated_confidence(answer: Dict[str, Any]) -> Optional[float]:+ """Extract calibrated confidence score from a Laya decision answer, or None if unusable."""+ return _gate_confidence(answer or {})@dataclass@@ -262,7 +262,20 @@ class LayaToolSelector:self.instructions = instructionsself.allow_direct_answer = bool(allow_direct_answer)self.direct_answer_description = direct_answer_description- self.confidence_threshold = confidence_threshold+ if confidence_threshold is not None:+ if (+ isinstance(confidence_threshold, bool)+ or not isinstance(confidence_threshold, (int, float))+ or not math.isfinite(confidence_threshold)+ or confidence_threshold < 0.0+ or confidence_threshold > 1.0+ ):+ raise ValueError(+ f"confidence_threshold must be a float in [0.0, 1.0], got {confidence_threshold!r}"+ )+ self.confidence_threshold = float(confidence_threshold)+ else:+ self.confidence_threshold = Noneself.fallback_tool = fallback_toolself.agent = agentself.base_url = base_url@@ -271,6 +284,8 @@ class LayaToolSelector:self.max_len = max_lenself.head_max_len = head_max_lenself.lang = lang+ if min_confidence is not None:+ check_min_confidence(min_confidence)self.min_confidence = min_confidenceself.hooks = hooksself.on_predict_start = on_predict_start@@ -324,52 +339,111 @@ class LayaToolSelector:hooks_timeout=self.hooks_timeout,)- answer = result.get("answers", {}).get("tool_select", {})+ answers_dict = result.get("answers")+ if not isinstance(answers_dict, dict) or "tool_select" not in answers_dict:+ if self.fallback_tool is not None:+ fb_name, _ = _extract_tool_metadata(self.fallback_tool, -1)+ return ToolRouteDecision(+ tool=self.fallback_tool,+ tool_name=fb_name,+ tool_index=-1,+ confidence=0.0,+ probabilities={},+ reason="Missing or malformed decision response; routed to fallback tool",+ is_direct_answer=False,+ is_fallback=True,+ raw_decision=result,+ )+ raise RuntimeError(f"Malformed or missing decision response from Laya: {result!r}")++ answer = answers_dict["tool_select"]+ if not isinstance(answer, dict):+ if self.fallback_tool is not None:+ fb_name, _ = _extract_tool_metadata(self.fallback_tool, -1)+ return ToolRouteDecision(+ tool=self.fallback_tool,+ tool_name=fb_name,+ tool_index=-1,+ confidence=0.0,+ probabilities={},+ reason="Malformed decision answer; routed to fallback tool",+ is_direct_answer=False,+ is_fallback=True,+ raw_decision=result,+ )+ raise RuntimeError(f"Malformed decision answer from Laya: {answer!r}")+chosen_key = answer.get("choice")conf = _gated_confidence(answer)raw_probs = answer.get("probabilities", {})# Map internal keys back to tool names in probabilitiesprobabilities: Dict[str, float] = {}- for k, prob in raw_probs.items():- if k in lookup:- tool_info = lookup[k]- probabilities[tool_info[2]] = float(prob)- else:- probabilities[k] = float(prob)-- # Check confidence gating+ if isinstance(raw_probs, dict):+ for k, prob in raw_probs.items():+ if k in lookup:+ tool_info = lookup[k]+ probabilities[tool_info[2]] = float(prob)+ else:+ probabilities[k] = float(prob)++ # Check confidence gating: unusable, missing, or NaN confidence fails closedthreshold = self.confidence_threshold if self.confidence_threshold is not None else self.min_confidence- is_low_confidence = bool(answer.get("low_confidence")) or (threshold is not None and conf < threshold)+ is_low_confidence = (+ bool(answer.get("low_confidence"))+ or conf is None+ or (threshold is not None and conf < threshold)+ )if is_low_confidence:+ effective_conf = conf if conf is not None else 0.0if self.fallback_tool is not None:fb_name, _ = _extract_tool_metadata(self.fallback_tool, -1)return ToolRouteDecision(tool=self.fallback_tool,tool_name=fb_name,tool_index=-1,- confidence=conf,+ confidence=effective_conf,probabilities=probabilities,- reason=f"Confidence {conf:.3f} below threshold {threshold}; routed to fallback tool",+ reason=(+ f"Confidence {effective_conf:.3f} below threshold {threshold}; routed to fallback tool"+ if threshold is not None+ else "Unusable or low confidence decision; routed to fallback tool"+ ),is_direct_answer=False,is_fallback=True,raw_decision=result,)raise LayaLowConfidenceError(- f"Laya tool selection confidence {conf:.3f} is below threshold {threshold}",- confidence=conf,+ (+ f"Laya tool selection confidence {effective_conf:.3f} is below threshold {threshold}"+ if threshold is not None+ else "Laya tool selection returned unusable or low confidence."+ ),+ confidence=effective_conf,threshold=float(threshold) if threshold is not None else 0.0,raw_decision=result,)- if chosen_key not in lookup:- # Fallback to first available tool if unmapped key returned- first_key = next(iter(lookup))- idx, tool_obj, name = lookup[first_key]- else:- idx, tool_obj, name = lookup[chosen_key]+ if chosen_key is None or chosen_key not in lookup:+ if self.fallback_tool is not None:+ fb_name, _ = _extract_tool_metadata(self.fallback_tool, -1)+ return ToolRouteDecision(+ tool=self.fallback_tool,+ tool_name=fb_name,+ tool_index=-1,+ confidence=conf if conf is not None else 0.0,+ probabilities=probabilities,+ reason=f"Unknown or missing tool decision {chosen_key!r}; routed to fallback tool",+ is_direct_answer=False,+ is_fallback=True,+ raw_decision=result,+ )+ raise RuntimeError(+ f"Unknown or missing tool decision from Laya: choice {chosen_key!r} not in candidate tools."+ )+ idx, tool_obj, name = lookup[chosen_key]is_direct = (idx == -1)reason = ("Direct answer requested; no tool required"diff --git a/tests/test_tools.py b/tests/test_tools.pyindex 19705f2f..0371ca25 100644--- a/tests/test_tools.py+++ b/tests/test_tools.py@@ -189,6 +189,86 @@ check("threshold/fallback_tool_obj", decision_fb.tool, fallback_tool)check("threshold/fallback_index", decision_fb.tool_index, -1)+# --------------------------------------------------------------- 4b. Threshold Validation & Fail-Closed Behavior+# Validation of confidence_threshold to [0.0, 1.0]+for bad_thresh in (-0.1, 1.05, 2.0, True, False, float("nan"), "0.5"):+ try:+ LayaToolSelector(tools=tools_list, confidence_threshold=bad_thresh)+ FAIL.append(f"threshold_val/bad_{bad_thresh!r}: expected ValueError")+ except ValueError:+ PASS.append(f"threshold_val/rejects_{bad_thresh!r}")++check("threshold_val/accepts_zero", LayaToolSelector(tools=tools_list, confidence_threshold=0.0).confidence_threshold, 0.0)+check("threshold_val/accepts_one", LayaToolSelector(tools=tools_list, confidence_threshold=1.0).confidence_threshold, 1.0)+check("threshold_val/accepts_float", LayaToolSelector(tools=tools_list, confidence_threshold=0.75).confidence_threshold, 0.75)++# Fail-closed: empty answers object must raise rather than selecting tool_0+empty_agent = MockLayaAgent(lambda s, q: {"model": "laya-multilingual", "answers": {}})+selector_empty = LayaToolSelector(tools=tools_list, agent=empty_agent)+try:+ selector_empty.select("Query")+ FAIL.append("fail_closed/empty_answers: expected RuntimeError")+except RuntimeError as e:+ PASS.append("fail_closed/empty_answers_raises")+ check_true("fail_closed/empty_answers_msg", "Malformed or missing decision response" in str(e))++# Fail-closed with fallback: empty answers routes to fallback tool+selector_empty_fb = LayaToolSelector(tools=tools_list, fallback_tool=fallback_tool, agent=empty_agent)+dec_empty_fb = selector_empty_fb.select("Query")+check("fail_closed/empty_fallback_used", dec_empty_fb.is_fallback, True)+check("fail_closed/empty_fallback_name", dec_empty_fb.tool_name, "general_llm_fallback")++# Fail-closed: missing / unknown choice key+unknown_choice_agent = MockLayaAgent(lambda s, q: {"model": "laya-multilingual", "answers": {"tool_select": {"choice": "unknown_tool", "confidence": 0.99}}})+selector_unknown = LayaToolSelector(tools=tools_list, agent=unknown_choice_agent)+try:+ selector_unknown.select("Query")+ FAIL.append("fail_closed/unknown_choice: expected RuntimeError")+except RuntimeError as e:+ PASS.append("fail_closed/unknown_choice_raises")+ check_true("fail_closed/unknown_choice_msg", "choice 'unknown_tool' not in candidate tools" in str(e))++selector_unknown_fb = LayaToolSelector(tools=tools_list, fallback_tool=fallback_tool, agent=unknown_choice_agent)+dec_unknown_fb = selector_unknown_fb.select("Query")+check("fail_closed/unknown_choice_fallback_used", dec_unknown_fb.is_fallback, True)++# Fail-closed: unusable / None / NaN confidence must fail closed+nan_conf_agent = MockLayaAgent(lambda s, q: {+ "model": "laya-multilingual",+ "answers": {+ "tool_select": {+ "choice": "tool_0",+ "confidence": float("nan"),+ "answer_confidence": None,+ }+ }+})+selector_nan = LayaToolSelector(tools=tools_list, agent=nan_conf_agent)+try:+ selector_nan.select("Query")+ FAIL.append("fail_closed/nan_confidence: expected LayaLowConfidenceError")+except LayaLowConfidenceError:+ PASS.append("fail_closed/nan_confidence_raises")++selector_nan_fb = LayaToolSelector(tools=tools_list, fallback_tool=fallback_tool, agent=nan_conf_agent)+dec_nan_fb = selector_nan_fb.select("Query")+check("fail_closed/nan_confidence_fallback_used", dec_nan_fb.is_fallback, True)++# Fail-closed protects executable call(): never runs tool_0 on missing/malformed/unknown decisions+tool_exec_counter = {"calls": 0}+def dangerous_tool():+ tool_exec_counter["calls"] += 1+ return "executed"++selector_safe_call = LayaToolSelector(tools=[dangerous_tool], agent=empty_agent)+try:+ selector_safe_call.call("Query")+ FAIL.append("fail_closed/call_should_not_execute: expected RuntimeError")+except RuntimeError:+ PASS.append("fail_closed/call_prevented_execution")+check("fail_closed/tool_not_called", tool_exec_counter["calls"], 0)++# --------------------------------------------------------------- 5. Direct Execution (call)def add_numbers(a: int, b: int) -> int:"""Add two numbers together."""