Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(integrations): gate thresholds on answer_confidence, never entropy confidence
Aashish4e552251aa20
diff --git a/laya/integrations/crewai.py b/laya/integrations/crewai.pyindex 435d96c1..808efadd 100644--- a/laya/integrations/crewai.py+++ b/laya/integrations/crewai.py@@ -56,6 +56,7 @@ class LayaTaskGuardError(ValueError):# them came to forward only `model`.from ._controls import budget_kwargs as _budget_kwargs, hook_kwargs as _hook_kwargs # noqa: E402from ._controls import predict_kwargs as _predict_kwargs, reject_remote_hooks as _reject_remote_hooks # noqa: E402+from ..confidence import answer_confidence_value as _answer_confidence_value # noqa: E402# One class for every integration, so `except LayaLowConfidenceError` catches all of them.from ._errors import LayaLowConfidenceError # noqa: E402@@ -307,7 +308,11 @@ class LayaCrewRouter:ans = res.get("answers", {}).get("delegation", {})chosen_key = ans.get("choice")- conf = ans.get("answer_confidence", ans.get("confidence", 1.0))+ # The calibrated number core gates on, never the entropy field: `confidence` is on a+ # different scale (see `laya.confidence`), so a threshold on one must not read the other.+ conf = _answer_confidence_value(ans)+ if conf is None:+ conf = 1.0# Map agent_i back to integer index ichosen_idx: int = 0diff --git a/laya/integrations/langchain.py b/laya/integrations/langchain.pyindex d6282877..de96883a 100644--- a/laya/integrations/langchain.py+++ b/laya/integrations/langchain.py@@ -29,6 +29,7 @@ except ImportError:# The per-call control rules live in `._controls` because the CrewAI and LlamaIndex wrappers end# at the same `predict` call and the same laya-serve body; keeping the rule in three places is how# two of them ended up forwarding only `model`.+from ..confidence import answer_confidence_value as _answer_confidence_valuefrom ._controls import budget_kwargs as _budget_kwargs, hook_kwargs as _hook_kwargsfrom ._controls import predict_kwargs as _predict_kwargsfrom ._controls import reject_remote_hooks as _reject_remote_hooks@@ -466,9 +467,11 @@ class LayaRouter(_BatchedRunnable, RunnableSerializable):self.last_decision = resultans = result["answers"][self.question_id]choice = ans["choice"]- confidence = ans.get("answer_confidence")+ # The calibrated number core gates on, never the entropy field: `confidence` is on a+ # different scale (see `laya.confidence`), so a threshold on one must not read the other.+ confidence = _answer_confidence_value(ans)if confidence is None:- confidence = ans.get("confidence", 1.0)+ confidence = 1.0if self.confidence_threshold > 0.0 and confidence < self.confidence_threshold:if self.fallback is not None:diff --git a/laya/integrations/llamaindex.py b/laya/integrations/llamaindex.pyindex 260df8f3..6706621c 100644--- a/laya/integrations/llamaindex.py+++ b/laya/integrations/llamaindex.py@@ -68,6 +68,19 @@ from ._controls import predict_kwargs as _predict_kwargs, reject_remote_hooks as# One class for every integration, so `except LayaLowConfidenceError` catches all of them.from ._errors import LayaLowConfidenceError # noqa: E402+from ..confidence import answer_confidence_value as _answer_confidence_value # noqa: E402+++def _gated_confidence(answer: Dict[str, Any]) -> float:+ """The threshold number for one answer: the calibrated field, defaulting to 1.0.++ Every selector and router in this module gates `confidence_threshold` on this, and+ they must all read the same quantity -- `answer_confidence`, the calibrated number+ core gates on, never the entropy `confidence` (see `laya.confidence`). An answer that+ reports no usable confidence is treated as fully confident, exactly as before.+ """+ conf = _answer_confidence_value(answer or {})+ return conf if conf is not None else 1.0def _extract_query_str(query: Union[str, QueryBundle, Any]) -> str:@@ -292,7 +305,7 @@ class LayaSingleSelector(BaseSelector):ans = res.get("answers", {}).get("selector", {})chosen_key = ans.get("choice")- conf = ans.get("answer_confidence", ans.get("confidence", 1.0))+ conf = _gated_confidence(ans)# Map "choice_i" back to index ichosen_idx: int = 0@@ -578,7 +591,7 @@ class LayaQueryRouter:ans = res.get("answers", {}).get("route", {})chosen = ans.get("choice")- conf = ans.get("answer_confidence", ans.get("confidence", 1.0))+ conf = _gated_confidence(ans)if self.confidence_threshold > 0.0 and conf < self.confidence_threshold:if self.fallback_key:diff --git a/tests/test_crewai.py b/tests/test_crewai.pyindex bebbaa32..2212e97d 100644--- a/tests/test_crewai.py+++ b/tests/test_crewai.py@@ -452,6 +452,44 @@ except Exception as exc:check_true("controls/guard remote refuses hooks", False, type(exc).__name__)+# --------------------------------------------------------------- Confidence source+# The threshold gates the calibrated field, never the entropy one: `confidence` is on a+# different scale (see `laya.confidence`), so the two must not be interchangeable here.+++class DisagreeingAgent(MockLayaAgent):+ """Answers whose two confidence fields disagree, on purpose."""+++def _disagreeing_router(**kwargs):+ def response(state, questions):+ return {"model": "mock-crew-router",+ "answers": {"delegation": {"choice": "agent_0", "confidence": 0.95,+ "answer_confidence": 0.4}}}+ return LayaCrewRouter(agent=DisagreeingAgent(response), **kwargs)+++# Below the calibrated threshold but above the entropy one: gates on the calibrated number.+low = _disagreeing_router(confidence_threshold=0.80, fallback_agent_index=1)+decided = low.route("anything", agents)+check("gate/reads calibrated not entropy", decided.agent_index, 1)+try:+ _disagreeing_router(confidence_threshold=0.80, raise_on_low_confidence=True).route("anything",+ agents)+ check_true("gate/raises on the calibrated number", False, "no error raised")+except LayaLowConfidenceError as err:+ check("gate/error carries the calibrated number", err.confidence, 0.4)++# An answer with no usable confidence keeps the old behaviour: treated as fully confident.+def _silent(state, questions):+ return {"model": "mock-crew-router", "answers": {"delegation": {"choice": "agent_2"}}}+++kept = LayaCrewRouter(agent=DisagreeingAgent(_silent), confidence_threshold=0.80).route(+ "anything", agents)+check("gate/missing confidence still passes", kept.agent_index, 2)++# --------------------------------------------------------------- Results Summaryprint(f"PASS: {len(PASS)}")print(f"FAIL: {len(FAIL)}")diff --git a/tests/test_langchain.py b/tests/test_langchain.pyindex e56c09ad..43acc24f 100644--- a/tests/test_langchain.py+++ b/tests/test_langchain.py@@ -226,6 +226,7 @@ def mock_router_response(state, questions):"choice": choice,"probabilities": {"billing": conf, "technical": 1.0 - conf},"confidence": conf,+ "answer_confidence": conf,}},}diff --git a/tests/test_llamaindex.py b/tests/test_llamaindex.pyindex dcfbb4ce..d6ae3b5a 100644--- a/tests/test_llamaindex.py+++ b/tests/test_llamaindex.py@@ -551,6 +551,60 @@ check_true("controls/forwards the caller's objects",_seen_hooks is _sentinel_hooks and _seen_hooks[0] is _sentinel_hooks[0], repr(_seen_hooks))+# --------------------------------------------------------------- Confidence source+# Every gate in this module reads the same number: the calibrated `answer_confidence`+# core gates on, never the entropy `confidence` (see `laya.confidence`). The two are on+# different scales, so they must not be interchangeable here.+++class DisagreeingSelectorAgent(RecordingAgent):+ """Answers whose two confidence fields disagree, on purpose."""++ def __init__(self, response_fn=None):+ super().__init__()+ self.response_fn = response_fn++ def predict(self, state, questions, **kwargs):+ self.calls.append(kwargs)+ if self.response_fn is not None:+ return self.response_fn(state, questions)+ return super().predict(state, questions, **kwargs)+++def _disagree(choice_answer):+ def response(state, questions):+ return {"model": "m",+ "answers": {"selector": dict({"choice": "choice_0"}, **choice_answer),+ "route": {"choice": "sql", "confidence": 0.9, "answer_confidence": 0.9}}}+ return DisagreeingSelectorAgent(response)+++# Below the calibrated threshold but above the entropy one: gates on the calibrated number.+low_agent = _disagree({"confidence": 0.95, "answer_confidence": 0.4})+low_sel = LayaSingleSelector(agent=low_agent, confidence_threshold=0.80, fallback_index=1).select(+ tools, "anything")+check("gate/reads calibrated not entropy",+ low_sel.selections[0].index if hasattr(low_sel, "selections") else low_sel, 1)+try:+ LayaSingleSelector(agent=_disagree({"confidence": 0.95, "answer_confidence": 0.4}),+ confidence_threshold=0.80,+ raise_on_low_confidence=True).select(tools, "anything")+ check_true("gate/raises on the calibrated number", False, "no error raised")+except LayaLowConfidenceError as err:+ check("gate/error carries the calibrated number", err.confidence, 0.4)++# An answer with no usable confidence keeps the old behaviour: treated as fully confident.+def _silent_selector(state, questions):+ return {"model": "m",+ "answers": {"selector": {"choice": "choice_2"},+ "route": {"choice": "sql", "confidence": 0.9, "answer_confidence": 0.9}}}+++kept = LayaSingleSelector(agent=DisagreeingSelectorAgent(_silent_selector)).select(tools, "anything")+check("gate/missing confidence still passes",+ kept.selections[0].index if hasattr(kept, "selections") else kept, 2)++# --------------------------------------------------------------- Results Summaryprint(f"PASS: {len(PASS)}")print(f"FAIL: {len(FAIL)}")