Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(integrations): fail closed on missing or unusable tool decisions and validate confidence threshold

LEVELING2108305ae8a8e8fa
diff --git a/laya/integrations/tools.py b/laya/integrations/tools.py
index dbd474f5..b8d8d017 100644
--- a/laya/integrations/tools.py
+++ b/laya/integrations/tools.py
@@ -14,6 +14,7 @@ from __future__ import annotations
import asyncio
import json
+import math
import threading
import urllib.error
import urllib.parse
@@ -28,13 +29,12 @@ from ._controls import hook_kwargs as _hook_kwargs
from ._controls import predict_kwargs as _predict_kwargs
from ._controls import reject_remote_hooks as _reject_remote_hooks
from ._errors import LayaLowConfidenceError
-from ..confidence import _gate_confidence
+from ..confidence import _gate_confidence, check_min_confidence
-def _gated_confidence(answer: Dict[str, Any]) -> float:
- """Extract calibrated confidence score from a Laya decision answer."""
- conf = _gate_confidence(answer or {})
- return conf if conf is not None else 1.0
+def _gated_confidence(answer: Dict[str, Any]) -> Optional[float]:
+ """Extract calibrated confidence score from a Laya decision answer, or None if unusable."""
+ return _gate_confidence(answer or {})
@dataclass
@@ -262,7 +262,20 @@ class LayaToolSelector:
self.instructions = instructions
self.allow_direct_answer = bool(allow_direct_answer)
self.direct_answer_description = direct_answer_description
- self.confidence_threshold = confidence_threshold
+ if confidence_threshold is not None:
+ if (
+ isinstance(confidence_threshold, bool)
+ or not isinstance(confidence_threshold, (int, float))
+ or not math.isfinite(confidence_threshold)
+ or confidence_threshold < 0.0
+ or confidence_threshold > 1.0
+ ):
+ raise ValueError(
+ f"confidence_threshold must be a float in [0.0, 1.0], got {confidence_threshold!r}"
+ )
+ self.confidence_threshold = float(confidence_threshold)
+ else:
+ self.confidence_threshold = None
self.fallback_tool = fallback_tool
self.agent = agent
self.base_url = base_url
@@ -271,6 +284,8 @@ class LayaToolSelector:
self.max_len = max_len
self.head_max_len = head_max_len
self.lang = lang
+ if min_confidence is not None:
+ check_min_confidence(min_confidence)
self.min_confidence = min_confidence
self.hooks = hooks
self.on_predict_start = on_predict_start
@@ -324,52 +339,111 @@ class LayaToolSelector:
hooks_timeout=self.hooks_timeout,
)
- answer = result.get("answers", {}).get("tool_select", {})
+ answers_dict = result.get("answers")
+ if not isinstance(answers_dict, dict) or "tool_select" not in answers_dict:
+ if self.fallback_tool is not None:
+ fb_name, _ = _extract_tool_metadata(self.fallback_tool, -1)
+ return ToolRouteDecision(
+ tool=self.fallback_tool,
+ tool_name=fb_name,
+ tool_index=-1,
+ confidence=0.0,
+ probabilities={},
+ reason="Missing or malformed decision response; routed to fallback tool",
+ is_direct_answer=False,
+ is_fallback=True,
+ raw_decision=result,
+ )
+ raise RuntimeError(f"Malformed or missing decision response from Laya: {result!r}")
+
+ answer = answers_dict["tool_select"]
+ if not isinstance(answer, dict):
+ if self.fallback_tool is not None:
+ fb_name, _ = _extract_tool_metadata(self.fallback_tool, -1)
+ return ToolRouteDecision(
+ tool=self.fallback_tool,
+ tool_name=fb_name,
+ tool_index=-1,
+ confidence=0.0,
+ probabilities={},
+ reason="Malformed decision answer; routed to fallback tool",
+ is_direct_answer=False,
+ is_fallback=True,
+ raw_decision=result,
+ )
+ raise RuntimeError(f"Malformed decision answer from Laya: {answer!r}")
+
chosen_key = answer.get("choice")
conf = _gated_confidence(answer)
raw_probs = answer.get("probabilities", {})
# Map internal keys back to tool names in probabilities
probabilities: Dict[str, float] = {}
- for k, prob in raw_probs.items():
- if k in lookup:
- tool_info = lookup[k]
- probabilities[tool_info[2]] = float(prob)
- else:
- probabilities[k] = float(prob)
-
- # Check confidence gating
+ if isinstance(raw_probs, dict):
+ for k, prob in raw_probs.items():
+ if k in lookup:
+ tool_info = lookup[k]
+ probabilities[tool_info[2]] = float(prob)
+ else:
+ probabilities[k] = float(prob)
+
+ # Check confidence gating: unusable, missing, or NaN confidence fails closed
threshold = self.confidence_threshold if self.confidence_threshold is not None else self.min_confidence
- is_low_confidence = bool(answer.get("low_confidence")) or (threshold is not None and conf < threshold)
+ is_low_confidence = (
+ bool(answer.get("low_confidence"))
+ or conf is None
+ or (threshold is not None and conf < threshold)
+ )
if is_low_confidence:
+ effective_conf = conf if conf is not None else 0.0
if self.fallback_tool is not None:
fb_name, _ = _extract_tool_metadata(self.fallback_tool, -1)
return ToolRouteDecision(
tool=self.fallback_tool,
tool_name=fb_name,
tool_index=-1,
- confidence=conf,
+ confidence=effective_conf,
probabilities=probabilities,
- reason=f"Confidence {conf:.3f} below threshold {threshold}; routed to fallback tool",
+ reason=(
+ f"Confidence {effective_conf:.3f} below threshold {threshold}; routed to fallback tool"
+ if threshold is not None
+ else "Unusable or low confidence decision; routed to fallback tool"
+ ),
is_direct_answer=False,
is_fallback=True,
raw_decision=result,
)
raise LayaLowConfidenceError(
- f"Laya tool selection confidence {conf:.3f} is below threshold {threshold}",
- confidence=conf,
+ (
+ f"Laya tool selection confidence {effective_conf:.3f} is below threshold {threshold}"
+ if threshold is not None
+ else "Laya tool selection returned unusable or low confidence."
+ ),
+ confidence=effective_conf,
threshold=float(threshold) if threshold is not None else 0.0,
raw_decision=result,
)
- if chosen_key not in lookup:
- # Fallback to first available tool if unmapped key returned
- first_key = next(iter(lookup))
- idx, tool_obj, name = lookup[first_key]
- else:
- idx, tool_obj, name = lookup[chosen_key]
+ if chosen_key is None or chosen_key not in lookup:
+ if self.fallback_tool is not None:
+ fb_name, _ = _extract_tool_metadata(self.fallback_tool, -1)
+ return ToolRouteDecision(
+ tool=self.fallback_tool,
+ tool_name=fb_name,
+ tool_index=-1,
+ confidence=conf if conf is not None else 0.0,
+ probabilities=probabilities,
+ reason=f"Unknown or missing tool decision {chosen_key!r}; routed to fallback tool",
+ is_direct_answer=False,
+ is_fallback=True,
+ raw_decision=result,
+ )
+ raise RuntimeError(
+ f"Unknown or missing tool decision from Laya: choice {chosen_key!r} not in candidate tools."
+ )
+ idx, tool_obj, name = lookup[chosen_key]
is_direct = (idx == -1)
reason = (
"Direct answer requested; no tool required"
diff --git a/tests/test_tools.py b/tests/test_tools.py
index 19705f2f..0371ca25 100644
--- a/tests/test_tools.py
+++ b/tests/test_tools.py
@@ -189,6 +189,86 @@ check("threshold/fallback_tool_obj", decision_fb.tool, fallback_tool)
check("threshold/fallback_index", decision_fb.tool_index, -1)
+# --------------------------------------------------------------- 4b. Threshold Validation & Fail-Closed Behavior
+# Validation of confidence_threshold to [0.0, 1.0]
+for bad_thresh in (-0.1, 1.05, 2.0, True, False, float("nan"), "0.5"):
+ try:
+ LayaToolSelector(tools=tools_list, confidence_threshold=bad_thresh)
+ FAIL.append(f"threshold_val/bad_{bad_thresh!r}: expected ValueError")
+ except ValueError:
+ PASS.append(f"threshold_val/rejects_{bad_thresh!r}")
+
+check("threshold_val/accepts_zero", LayaToolSelector(tools=tools_list, confidence_threshold=0.0).confidence_threshold, 0.0)
+check("threshold_val/accepts_one", LayaToolSelector(tools=tools_list, confidence_threshold=1.0).confidence_threshold, 1.0)
+check("threshold_val/accepts_float", LayaToolSelector(tools=tools_list, confidence_threshold=0.75).confidence_threshold, 0.75)
+
+# Fail-closed: empty answers object must raise rather than selecting tool_0
+empty_agent = MockLayaAgent(lambda s, q: {"model": "laya-multilingual", "answers": {}})
+selector_empty = LayaToolSelector(tools=tools_list, agent=empty_agent)
+try:
+ selector_empty.select("Query")
+ FAIL.append("fail_closed/empty_answers: expected RuntimeError")
+except RuntimeError as e:
+ PASS.append("fail_closed/empty_answers_raises")
+ check_true("fail_closed/empty_answers_msg", "Malformed or missing decision response" in str(e))
+
+# Fail-closed with fallback: empty answers routes to fallback tool
+selector_empty_fb = LayaToolSelector(tools=tools_list, fallback_tool=fallback_tool, agent=empty_agent)
+dec_empty_fb = selector_empty_fb.select("Query")
+check("fail_closed/empty_fallback_used", dec_empty_fb.is_fallback, True)
+check("fail_closed/empty_fallback_name", dec_empty_fb.tool_name, "general_llm_fallback")
+
+# Fail-closed: missing / unknown choice key
+unknown_choice_agent = MockLayaAgent(lambda s, q: {"model": "laya-multilingual", "answers": {"tool_select": {"choice": "unknown_tool", "confidence": 0.99}}})
+selector_unknown = LayaToolSelector(tools=tools_list, agent=unknown_choice_agent)
+try:
+ selector_unknown.select("Query")
+ FAIL.append("fail_closed/unknown_choice: expected RuntimeError")
+except RuntimeError as e:
+ PASS.append("fail_closed/unknown_choice_raises")
+ check_true("fail_closed/unknown_choice_msg", "choice 'unknown_tool' not in candidate tools" in str(e))
+
+selector_unknown_fb = LayaToolSelector(tools=tools_list, fallback_tool=fallback_tool, agent=unknown_choice_agent)
+dec_unknown_fb = selector_unknown_fb.select("Query")
+check("fail_closed/unknown_choice_fallback_used", dec_unknown_fb.is_fallback, True)
+
+# Fail-closed: unusable / None / NaN confidence must fail closed
+nan_conf_agent = MockLayaAgent(lambda s, q: {
+ "model": "laya-multilingual",
+ "answers": {
+ "tool_select": {
+ "choice": "tool_0",
+ "confidence": float("nan"),
+ "answer_confidence": None,
+ }
+ }
+})
+selector_nan = LayaToolSelector(tools=tools_list, agent=nan_conf_agent)
+try:
+ selector_nan.select("Query")
+ FAIL.append("fail_closed/nan_confidence: expected LayaLowConfidenceError")
+except LayaLowConfidenceError:
+ PASS.append("fail_closed/nan_confidence_raises")
+
+selector_nan_fb = LayaToolSelector(tools=tools_list, fallback_tool=fallback_tool, agent=nan_conf_agent)
+dec_nan_fb = selector_nan_fb.select("Query")
+check("fail_closed/nan_confidence_fallback_used", dec_nan_fb.is_fallback, True)
+
+# Fail-closed protects executable call(): never runs tool_0 on missing/malformed/unknown decisions
+tool_exec_counter = {"calls": 0}
+def dangerous_tool():
+ tool_exec_counter["calls"] += 1
+ return "executed"
+
+selector_safe_call = LayaToolSelector(tools=[dangerous_tool], agent=empty_agent)
+try:
+ selector_safe_call.call("Query")
+ FAIL.append("fail_closed/call_should_not_execute: expected RuntimeError")
+except RuntimeError:
+ PASS.append("fail_closed/call_prevented_execution")
+check("fail_closed/tool_not_called", tool_exec_counter["calls"], 0)
+
+
# --------------------------------------------------------------- 5. Direct Execution (call)
def add_numbers(a: int, b: int) -> int:
"""Add two numbers together."""