Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(confidence): clear stale low_confidence flag on re-evaluation (#910)

Sarthak Pandeye948dbde65c8
diff --git a/laya/confidence.py b/laya/confidence.py
index 57401b3c..d3eaec73 100644
--- a/laya/confidence.py
+++ b/laya/confidence.py
@@ -134,7 +134,9 @@ def flag_low_confidence(results: List[Dict[str, Any]], min_confidence: float) ->
Reads `answer_confidence` (`max(p)`, the quantity the calibration figures describe and the one
that does not drift with the number of options), falling back to `confidence` if
`answer_confidence` is absent.
- The raw answer and confidence stay intact; `low_confidence: True` is added.
+ The raw answer and confidence stay intact; `low_confidence: True` is added when the answer
+ falls below the threshold, and removed if a previously-flagged answer now clears it (e.g.
+ when a result dict is reused or re-evaluated with a different threshold).
`min_confidence` is either a float (one threshold for every answer) or a per-bucket mapping
(#394), in which case each answer is gated at the threshold of its own option-count bucket via
@@ -152,10 +154,13 @@ def flag_low_confidence(results: List[Dict[str, Any]], min_confidence: float) ->
continue
conf = _gate_confidence(a)
if conf is None:
+ a.pop("low_confidence", None)
continue
thr = resolve_min_confidence(a, min_confidence) if is_map else min_confidence
if conf < thr:
a["low_confidence"] = True
+ else:
+ a.pop("low_confidence", None)
#: The states :func:`apply_confidence_gate` reports, and the only ones. There is deliberately no
@@ -214,6 +219,8 @@ def apply_confidence_gate(results: List[Dict[str, Any]], min_confidence: Optiona
for a in answers.values():
if not isinstance(a, dict):
continue
+ if not is_map and min_confidence == 0.0:
+ a.pop("low_confidence", None)
if a.get("low_confidence"):
a["abstention"] = GATE_ABSTAINED
elif _gate_confidence(a) is None:
diff --git a/tests/test_confidence.py b/tests/test_confidence.py
index 2ef047d1..21171522 100644
--- a/tests/test_confidence.py
+++ b/tests/test_confidence.py
@@ -260,6 +260,54 @@ apply_confidence_gate([None, {"answers": None}, {"answers": [1, 2]}], 0.5)
apply_confidence_gate([], None)
PASS.append("gate/non-dict results are skipped without raising")
+# --------------------------------------------------------------- BUG-005 regression: stale low_confidence across reuse (#910)
+# `flag_low_confidence` and `apply_confidence_gate` used to only set `low_confidence: True` and
+# never clear it. When a result dict was re-evaluated with a more permissive threshold (or 0.0),
+# the stale flag remained, causing downstream callers to treat the answer as permanently abstained.
+
+# 1. flag_low_confidence clears stale flag on re-evaluation
+reuse_res = [{
+ "answers": {
+ "intent": {"choice": "refund", "answer_confidence": 0.8},
+ }
+}]
+reuse_ans = reuse_res[0]["answers"]["intent"]
+flag_low_confidence(reuse_res, 0.9)
+check_true("stale/flag strict threshold sets flag", reuse_ans.get("low_confidence") is True)
+flag_low_confidence(reuse_res, 0.5)
+check_true("stale/flag permissive threshold clears flag", "low_confidence" not in reuse_ans)
+
+clean_res = [{"answers": {"q": {"answer_confidence": 0.95}}}]
+flag_low_confidence(clean_res, 0.5)
+check_true("stale/unflagged answer stays clean", "low_confidence" not in clean_res[0]["answers"]["q"])
+
+# 2. apply_confidence_gate clears stale flag and updates abstention state on re-evaluation
+gate_reuse = [{
+ "answers": {
+ "intent": {"choice": "refund", "answer_confidence": 0.8},
+ }
+}]
+g_ans = gate_reuse[0]["answers"]["intent"]
+apply_confidence_gate(gate_reuse, 0.9)
+check("stale/gate strict threshold abstains", g_ans["abstention"], GATE_ABSTAINED)
+check_true("stale/gate strict threshold sets low_conf", g_ans.get("low_confidence") is True)
+
+apply_confidence_gate(gate_reuse, 0.5)
+check("stale/gate permissive threshold passes", g_ans["abstention"], GATE_PASSED)
+check_true("stale/gate permissive threshold clears low_conf", "low_confidence" not in g_ans)
+check("stale/gate echoes updated threshold", g_ans["abstention_threshold"], 0.5)
+
+apply_confidence_gate(gate_reuse, 0.0)
+check("stale/gate 0.0 passes", g_ans["abstention"], GATE_PASSED)
+check_true("stale/gate 0.0 clears low_conf", "low_confidence" not in g_ans)
+
+# 3. unusable confidence clears stale low_confidence and marks unevaluated
+odd_reuse = [{"answers": {"q": {"choice": "x", "answer_confidence": 0.2, "low_confidence": True}}}]
+odd_reuse[0]["answers"]["q"]["answer_confidence"] = float("nan")
+apply_confidence_gate(odd_reuse, 0.5)
+check("stale/unusable conf marks unevaluated", odd_reuse[0]["answers"]["q"]["abstention"], GATE_UNEVALUATED)
+check_true("stale/unusable conf clears stale low_conf", "low_confidence" not in odd_reuse[0]["answers"]["q"])
+
# --------------------------------------------------------------- the operator's three questions
# End to end through a real call site, weight-free. `decide` forwards to any runner with a
# `predict`, so this exercises `laya/structured.py`'s own gate call site rather than the helper