Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(integrations): gate thresholds on answer_confidence, never entropy confidence

Aashish4e552251aa20
diff --git a/laya/integrations/crewai.py b/laya/integrations/crewai.py
index 435d96c1..808efadd 100644
--- a/laya/integrations/crewai.py
+++ b/laya/integrations/crewai.py
@@ -56,6 +56,7 @@ class LayaTaskGuardError(ValueError):
# them came to forward only `model`.
from ._controls import budget_kwargs as _budget_kwargs, hook_kwargs as _hook_kwargs # noqa: E402
from ._controls import predict_kwargs as _predict_kwargs, reject_remote_hooks as _reject_remote_hooks # noqa: E402
+from ..confidence import answer_confidence_value as _answer_confidence_value # noqa: E402
# One class for every integration, so `except LayaLowConfidenceError` catches all of them.
from ._errors import LayaLowConfidenceError # noqa: E402
@@ -307,7 +308,11 @@ class LayaCrewRouter:
ans = res.get("answers", {}).get("delegation", {})
chosen_key = ans.get("choice")
- conf = ans.get("answer_confidence", ans.get("confidence", 1.0))
+ # The calibrated number core gates on, never the entropy field: `confidence` is on a
+ # different scale (see `laya.confidence`), so a threshold on one must not read the other.
+ conf = _answer_confidence_value(ans)
+ if conf is None:
+ conf = 1.0
# Map agent_i back to integer index i
chosen_idx: int = 0
diff --git a/laya/integrations/langchain.py b/laya/integrations/langchain.py
index d6282877..de96883a 100644
--- a/laya/integrations/langchain.py
+++ b/laya/integrations/langchain.py
@@ -29,6 +29,7 @@ except ImportError:
# The per-call control rules live in `._controls` because the CrewAI and LlamaIndex wrappers end
# at the same `predict` call and the same laya-serve body; keeping the rule in three places is how
# two of them ended up forwarding only `model`.
+from ..confidence import answer_confidence_value as _answer_confidence_value
from ._controls import budget_kwargs as _budget_kwargs, hook_kwargs as _hook_kwargs
from ._controls import predict_kwargs as _predict_kwargs
from ._controls import reject_remote_hooks as _reject_remote_hooks
@@ -466,9 +467,11 @@ class LayaRouter(_BatchedRunnable, RunnableSerializable):
self.last_decision = result
ans = result["answers"][self.question_id]
choice = ans["choice"]
- confidence = ans.get("answer_confidence")
+ # The calibrated number core gates on, never the entropy field: `confidence` is on a
+ # different scale (see `laya.confidence`), so a threshold on one must not read the other.
+ confidence = _answer_confidence_value(ans)
if confidence is None:
- confidence = ans.get("confidence", 1.0)
+ confidence = 1.0
if self.confidence_threshold > 0.0 and confidence < self.confidence_threshold:
if self.fallback is not None:
diff --git a/laya/integrations/llamaindex.py b/laya/integrations/llamaindex.py
index 260df8f3..6706621c 100644
--- a/laya/integrations/llamaindex.py
+++ b/laya/integrations/llamaindex.py
@@ -68,6 +68,19 @@ from ._controls import predict_kwargs as _predict_kwargs, reject_remote_hooks as
# One class for every integration, so `except LayaLowConfidenceError` catches all of them.
from ._errors import LayaLowConfidenceError # noqa: E402
+from ..confidence import answer_confidence_value as _answer_confidence_value # noqa: E402
+
+
+def _gated_confidence(answer: Dict[str, Any]) -> float:
+ """The threshold number for one answer: the calibrated field, defaulting to 1.0.
+
+ Every selector and router in this module gates `confidence_threshold` on this, and
+ they must all read the same quantity -- `answer_confidence`, the calibrated number
+ core gates on, never the entropy `confidence` (see `laya.confidence`). An answer that
+ reports no usable confidence is treated as fully confident, exactly as before.
+ """
+ conf = _answer_confidence_value(answer or {})
+ return conf if conf is not None else 1.0
def _extract_query_str(query: Union[str, QueryBundle, Any]) -> str:
@@ -292,7 +305,7 @@ class LayaSingleSelector(BaseSelector):
ans = res.get("answers", {}).get("selector", {})
chosen_key = ans.get("choice")
- conf = ans.get("answer_confidence", ans.get("confidence", 1.0))
+ conf = _gated_confidence(ans)
# Map "choice_i" back to index i
chosen_idx: int = 0
@@ -578,7 +591,7 @@ class LayaQueryRouter:
ans = res.get("answers", {}).get("route", {})
chosen = ans.get("choice")
- conf = ans.get("answer_confidence", ans.get("confidence", 1.0))
+ conf = _gated_confidence(ans)
if self.confidence_threshold > 0.0 and conf < self.confidence_threshold:
if self.fallback_key:
diff --git a/tests/test_crewai.py b/tests/test_crewai.py
index bebbaa32..2212e97d 100644
--- a/tests/test_crewai.py
+++ b/tests/test_crewai.py
@@ -452,6 +452,44 @@ except Exception as exc:
check_true("controls/guard remote refuses hooks", False, type(exc).__name__)
+# --------------------------------------------------------------- Confidence source
+# The threshold gates the calibrated field, never the entropy one: `confidence` is on a
+# different scale (see `laya.confidence`), so the two must not be interchangeable here.
+
+
+class DisagreeingAgent(MockLayaAgent):
+ """Answers whose two confidence fields disagree, on purpose."""
+
+
+def _disagreeing_router(**kwargs):
+ def response(state, questions):
+ return {"model": "mock-crew-router",
+ "answers": {"delegation": {"choice": "agent_0", "confidence": 0.95,
+ "answer_confidence": 0.4}}}
+ return LayaCrewRouter(agent=DisagreeingAgent(response), **kwargs)
+
+
+# Below the calibrated threshold but above the entropy one: gates on the calibrated number.
+low = _disagreeing_router(confidence_threshold=0.80, fallback_agent_index=1)
+decided = low.route("anything", agents)
+check("gate/reads calibrated not entropy", decided.agent_index, 1)
+try:
+ _disagreeing_router(confidence_threshold=0.80, raise_on_low_confidence=True).route("anything",
+ agents)
+ check_true("gate/raises on the calibrated number", False, "no error raised")
+except LayaLowConfidenceError as err:
+ check("gate/error carries the calibrated number", err.confidence, 0.4)
+
+# An answer with no usable confidence keeps the old behaviour: treated as fully confident.
+def _silent(state, questions):
+ return {"model": "mock-crew-router", "answers": {"delegation": {"choice": "agent_2"}}}
+
+
+kept = LayaCrewRouter(agent=DisagreeingAgent(_silent), confidence_threshold=0.80).route(
+ "anything", agents)
+check("gate/missing confidence still passes", kept.agent_index, 2)
+
+
# --------------------------------------------------------------- Results Summary
print(f"PASS: {len(PASS)}")
print(f"FAIL: {len(FAIL)}")
diff --git a/tests/test_langchain.py b/tests/test_langchain.py
index e56c09ad..43acc24f 100644
--- a/tests/test_langchain.py
+++ b/tests/test_langchain.py
@@ -226,6 +226,7 @@ def mock_router_response(state, questions):
"choice": choice,
"probabilities": {"billing": conf, "technical": 1.0 - conf},
"confidence": conf,
+ "answer_confidence": conf,
}
},
}
diff --git a/tests/test_llamaindex.py b/tests/test_llamaindex.py
index dcfbb4ce..d6ae3b5a 100644
--- a/tests/test_llamaindex.py
+++ b/tests/test_llamaindex.py
@@ -551,6 +551,60 @@ check_true("controls/forwards the caller's objects",
_seen_hooks is _sentinel_hooks and _seen_hooks[0] is _sentinel_hooks[0], repr(_seen_hooks))
+# --------------------------------------------------------------- Confidence source
+# Every gate in this module reads the same number: the calibrated `answer_confidence`
+# core gates on, never the entropy `confidence` (see `laya.confidence`). The two are on
+# different scales, so they must not be interchangeable here.
+
+
+class DisagreeingSelectorAgent(RecordingAgent):
+ """Answers whose two confidence fields disagree, on purpose."""
+
+ def __init__(self, response_fn=None):
+ super().__init__()
+ self.response_fn = response_fn
+
+ def predict(self, state, questions, **kwargs):
+ self.calls.append(kwargs)
+ if self.response_fn is not None:
+ return self.response_fn(state, questions)
+ return super().predict(state, questions, **kwargs)
+
+
+def _disagree(choice_answer):
+ def response(state, questions):
+ return {"model": "m",
+ "answers": {"selector": dict({"choice": "choice_0"}, **choice_answer),
+ "route": {"choice": "sql", "confidence": 0.9, "answer_confidence": 0.9}}}
+ return DisagreeingSelectorAgent(response)
+
+
+# Below the calibrated threshold but above the entropy one: gates on the calibrated number.
+low_agent = _disagree({"confidence": 0.95, "answer_confidence": 0.4})
+low_sel = LayaSingleSelector(agent=low_agent, confidence_threshold=0.80, fallback_index=1).select(
+ tools, "anything")
+check("gate/reads calibrated not entropy",
+ low_sel.selections[0].index if hasattr(low_sel, "selections") else low_sel, 1)
+try:
+ LayaSingleSelector(agent=_disagree({"confidence": 0.95, "answer_confidence": 0.4}),
+ confidence_threshold=0.80,
+ raise_on_low_confidence=True).select(tools, "anything")
+ check_true("gate/raises on the calibrated number", False, "no error raised")
+except LayaLowConfidenceError as err:
+ check("gate/error carries the calibrated number", err.confidence, 0.4)
+
+# An answer with no usable confidence keeps the old behaviour: treated as fully confident.
+def _silent_selector(state, questions):
+ return {"model": "m",
+ "answers": {"selector": {"choice": "choice_2"},
+ "route": {"choice": "sql", "confidence": 0.9, "answer_confidence": 0.9}}}
+
+
+kept = LayaSingleSelector(agent=DisagreeingSelectorAgent(_silent_selector)).select(tools, "anything")
+check("gate/missing confidence still passes",
+ kept.selections[0].index if hasattr(kept, "selections") else kept, 2)
+
+
# --------------------------------------------------------------- Results Summary
print(f"PASS: {len(PASS)}")
print(f"FAIL: {len(FAIL)}")