Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(confidence): clear stale low_confidence flag on re-evaluation (#910)
Sarthak Pandeye948dbde65c8
diff --git a/laya/confidence.py b/laya/confidence.pyindex 57401b3c..d3eaec73 100644--- a/laya/confidence.py+++ b/laya/confidence.py@@ -134,7 +134,9 @@ def flag_low_confidence(results: List[Dict[str, Any]], min_confidence: float) ->Reads `answer_confidence` (`max(p)`, the quantity the calibration figures describe and the onethat does not drift with the number of options), falling back to `confidence` if`answer_confidence` is absent.- The raw answer and confidence stay intact; `low_confidence: True` is added.+ The raw answer and confidence stay intact; `low_confidence: True` is added when the answer+ falls below the threshold, and removed if a previously-flagged answer now clears it (e.g.+ when a result dict is reused or re-evaluated with a different threshold).`min_confidence` is either a float (one threshold for every answer) or a per-bucket mapping(#394), in which case each answer is gated at the threshold of its own option-count bucket via@@ -152,10 +154,13 @@ def flag_low_confidence(results: List[Dict[str, Any]], min_confidence: float) ->continueconf = _gate_confidence(a)if conf is None:+ a.pop("low_confidence", None)continuethr = resolve_min_confidence(a, min_confidence) if is_map else min_confidenceif conf < thr:a["low_confidence"] = True+ else:+ a.pop("low_confidence", None)#: The states :func:`apply_confidence_gate` reports, and the only ones. There is deliberately no@@ -214,6 +219,8 @@ def apply_confidence_gate(results: List[Dict[str, Any]], min_confidence: Optionafor a in answers.values():if not isinstance(a, dict):continue+ if not is_map and min_confidence == 0.0:+ a.pop("low_confidence", None)if a.get("low_confidence"):a["abstention"] = GATE_ABSTAINEDelif _gate_confidence(a) is None:diff --git a/tests/test_confidence.py b/tests/test_confidence.pyindex 2ef047d1..21171522 100644--- a/tests/test_confidence.py+++ b/tests/test_confidence.py@@ -260,6 +260,54 @@ apply_confidence_gate([None, {"answers": None}, {"answers": [1, 2]}], 0.5)apply_confidence_gate([], None)PASS.append("gate/non-dict results are skipped without raising")+# --------------------------------------------------------------- BUG-005 regression: stale low_confidence across reuse (#910)+# `flag_low_confidence` and `apply_confidence_gate` used to only set `low_confidence: True` and+# never clear it. When a result dict was re-evaluated with a more permissive threshold (or 0.0),+# the stale flag remained, causing downstream callers to treat the answer as permanently abstained.++# 1. flag_low_confidence clears stale flag on re-evaluation+reuse_res = [{+ "answers": {+ "intent": {"choice": "refund", "answer_confidence": 0.8},+ }+}]+reuse_ans = reuse_res[0]["answers"]["intent"]+flag_low_confidence(reuse_res, 0.9)+check_true("stale/flag strict threshold sets flag", reuse_ans.get("low_confidence") is True)+flag_low_confidence(reuse_res, 0.5)+check_true("stale/flag permissive threshold clears flag", "low_confidence" not in reuse_ans)++clean_res = [{"answers": {"q": {"answer_confidence": 0.95}}}]+flag_low_confidence(clean_res, 0.5)+check_true("stale/unflagged answer stays clean", "low_confidence" not in clean_res[0]["answers"]["q"])++# 2. apply_confidence_gate clears stale flag and updates abstention state on re-evaluation+gate_reuse = [{+ "answers": {+ "intent": {"choice": "refund", "answer_confidence": 0.8},+ }+}]+g_ans = gate_reuse[0]["answers"]["intent"]+apply_confidence_gate(gate_reuse, 0.9)+check("stale/gate strict threshold abstains", g_ans["abstention"], GATE_ABSTAINED)+check_true("stale/gate strict threshold sets low_conf", g_ans.get("low_confidence") is True)++apply_confidence_gate(gate_reuse, 0.5)+check("stale/gate permissive threshold passes", g_ans["abstention"], GATE_PASSED)+check_true("stale/gate permissive threshold clears low_conf", "low_confidence" not in g_ans)+check("stale/gate echoes updated threshold", g_ans["abstention_threshold"], 0.5)++apply_confidence_gate(gate_reuse, 0.0)+check("stale/gate 0.0 passes", g_ans["abstention"], GATE_PASSED)+check_true("stale/gate 0.0 clears low_conf", "low_confidence" not in g_ans)++# 3. unusable confidence clears stale low_confidence and marks unevaluated+odd_reuse = [{"answers": {"q": {"choice": "x", "answer_confidence": 0.2, "low_confidence": True}}}]+odd_reuse[0]["answers"]["q"]["answer_confidence"] = float("nan")+apply_confidence_gate(odd_reuse, 0.5)+check("stale/unusable conf marks unevaluated", odd_reuse[0]["answers"]["q"]["abstention"], GATE_UNEVALUATED)+check_true("stale/unusable conf clears stale low_conf", "low_confidence" not in odd_reuse[0]["answers"]["q"])+# --------------------------------------------------------------- the operator's three questions# End to end through a real call site, weight-free. `decide` forwards to any runner with a# `predict`, so this exercises `laya/structured.py`'s own gate call site rather than the helper