Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(confidence): per-option-count abstention thresholds (Closes #394)
AlKor133d762c4fdb75
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex c97f733d..a7da8150 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -149,6 +149,7 @@ jobs:python tests/test_shortlist_cosine.pypython tests/test_temperature_loading.pypython tests/test_confidence.py+ python tests/test_conformal_abstention.pypython tests/test_cli.pypython tests/test_cli_lang_guess.pypython tests/test_mcp.pydiff --git a/docs/reference/helpers.md b/docs/reference/helpers.mdindex eeda50a0..bdedf8be 100644--- a/docs/reference/helpers.md+++ b/docs/reference/helpers.md@@ -42,6 +42,10 @@::: laya.confidence.check_min_confidence+::: laya.confidence.check_min_confidence_map++::: laya.confidence.resolve_min_confidence+::: laya.confidence.flag_low_confidence::: laya.confidence.apply_confidence_gate@@ -62,6 +66,8 @@::: laya.calibrate.fit_temperature_map+::: laya.calibrate.fit_abstention_thresholds+::: laya.common.render_options::: laya.common.proper_rewarddiff --git a/laya/__init__.py b/laya/__init__.pyindex f1ffe8a8..a3ac90d6 100644--- a/laya/__init__.py+++ b/laya/__init__.py@@ -28,12 +28,15 @@ _LAZY_ATTRS = {"fit_temperatures": (".calibrate", "fit_temperatures"),"fit_one_temperature": (".calibrate", "fit_one_temperature"),"fit_temperature_map": (".calibrate", "fit_temperature_map"),+ "fit_abstention_thresholds": (".calibrate", "fit_abstention_thresholds"),"proper_reward": (".common", "proper_reward"),"td_lambda_targets": (".common", "td_lambda_targets"),"ece_score": (".common", "ece_score"),"answer_confidence": (".common", "answer_confidence"),"confidence_from_probs": (".common", "confidence_from_probs"),"check_min_confidence": (".confidence", "check_min_confidence"),+ "check_min_confidence_map": (".confidence", "check_min_confidence_map"),+ "resolve_min_confidence": (".confidence", "resolve_min_confidence"),"flag_low_confidence": (".confidence", "flag_low_confidence"),"apply_confidence_gate": (".confidence", "apply_confidence_gate"),"GATE_STATES": (".confidence", "GATE_STATES"),@@ -76,6 +79,7 @@ __all__ = ["fit_temperatures","fit_one_temperature","fit_temperature_map",+ "fit_abstention_thresholds","Router","RouteDecision","DEFAULT_MODELS",@@ -99,6 +103,8 @@ __all__ = ["answer_confidence","confidence_from_probs","check_min_confidence",+ "check_min_confidence_map",+ "resolve_min_confidence","flag_low_confidence","apply_confidence_gate","GATE_STATES",diff --git a/laya/calibrate.py b/laya/calibrate.pyindex 3906a5fe..e57a7c45 100644--- a/laya/calibrate.py+++ b/laya/calibrate.py@@ -258,6 +258,84 @@ def fit_temperature_map(records: Iterable, compute_ece: bool = False, seed: intfit_temperatures = fit_temperature_map+# Per-bucket floor for abstention thresholds. Lower than `MIN_BUCKET_N` (which governs temperature+# fitting): a threshold is a single order statistic of the calibrated confidences, so it stabilises+# on far fewer examples than an LBFGS temperature does. Buckets below this are omitted; the caller's+# scalar `min_confidence` (or the map's "default") covers them.+MIN_ABSTAIN_BUCKET_N = 100+++def _select_abstention_threshold(pairs: Sequence[Tuple[float, int]], target_error: float,+ conservative: bool) -> float:+ """Smallest confidence `tau` whose accepted set (`conf >= tau`) keeps error <= `target_error`.++ `pairs` are `(confidence, correct)`. Sweeping from the most confident down maximises coverage+ at the target risk (the selective-classification / split-conformal cut). `conservative` adds one+ pseudo-error so a bucket does not clear the gate on a lucky short run. Returns 1.0 when no cut+ holds the risk -- the bucket is too unreliable to accept anything short of a reported certainty.+ """+ ordered = sorted(pairs, key=lambda cc: cc[0], reverse=True)+ levels = sorted({c for c, _ in ordered}, reverse=True)+ n = len(ordered)+ n_acc = n_err = idx = 0+ best = None+ # Evaluate the error over the WHOLE accepted set {conf >= tau} at each distinct level, not+ # incrementally within a tie: a threshold accepts every answer at its own confidence, so a tie+ # group's errors must all be counted before the level is judged (else the cut sinks into a bad+ # cohort on its first few correct members).+ for tau in levels:+ while idx < n and ordered[idx][0] >= tau:+ n_acc += 1+ n_err += 0 if ordered[idx][1] else 1+ idx += 1+ rate = (n_err + 1.0) / (n_acc + 1.0) if conservative else (n_err / n_acc)+ if n_acc > 0 and rate <= target_error:+ best = tau+ if best is None:+ return 1.0+ return float(min(1.0, max(0.0, best)))+++def fit_abstention_thresholds(records: Iterable, temperature: Sequence[float],+ temperature_by_options: Dict[str, float], *,+ target_error: float = 0.10,+ min_bucket_n: int = MIN_ABSTAIN_BUCKET_N,+ conservative: bool = True) -> Dict[str, float]:+ """Fit a per-`temp_bucket` abstention threshold so a gate keeps a target error in every bucket.++ A single `min_confidence` does not transfer across option counts (#394): the calibrated+ confidence of a 2-option and a 12-option answer live on different scales, so one cut over- or+ under-abstains depending on the question. This fits one cut per bucket instead, keyed exactly+ like `temperature_by_options` (`common.temp_bucket`, e.g. ``"choice:3-5"``), and the result is a+ `min_confidence` map that :func:`laya.confidence.check_min_confidence` /+ :func:`laya.confidence.apply_confidence_gate` accept directly.++ `records` are the same `(qtype, logits, target[, k])` tuples `fit_temperature_map` consumes+ (`records_from_labeled` builds them). Confidence is the **calibrated** `max(p)` -- the logits are+ scaled by the fitted `temperature` / `temperature_by_options` first, so thresholds and the+ numbers the runtime reports are on the same scale. `target_error` is the tolerated error among+ accepted answers; `min_bucket_n` omits buckets too small to fit, and `conservative` adds a+ one-sample margin. The thresholds are empirical cuts on the calibration set, not a formal+ coverage guarantee -- validate on held-out data (`fit_temperature_map(..., compute_ece=True)`+ gives a held-out split) for a production gate.+ """+ if not 0.0 <= target_error <= 1.0:+ raise ValueError("target_error must be in [0.0, 1.0], got %r" % (target_error,))+ recs = _iter_records(records)+ by_bucket: Dict[str, List[Tuple[float, int]]] = {}+ for qt, z, t, k in recs:+ y = int(np.argmax(t[:k]))+ t_scale = temperature_by_options.get(temp_bucket(qt, k), temperature[qt])+ p = _softmax(z[:k], t_scale)+ by_bucket.setdefault(temp_bucket(qt, k), []).append((float(p.max()), int(int(p.argmax()) == y)))+ out: Dict[str, float] = {}+ for key, pairs in by_bucket.items():+ if len(pairs) < min_bucket_n:+ continue+ out[key] = _select_abstention_threshold(pairs, target_error, conservative)+ return out++def records_from_labeled(agent, pairs: Sequence) -> List[Record]:"""Collect CPU records from `(state, questions, targets)`.diff --git a/laya/confidence.py b/laya/confidence.pyindex 43b67ee5..57401b3c 100644--- a/laya/confidence.py+++ b/laya/confidence.py@@ -57,16 +57,77 @@ def _gate_confidence(answer: Dict[str, Any]) -> Optional[float]:return None-def check_min_confidence(v: Any) -> float:- """Validate opt-in abstention threshold `min_confidence` (#361).-- Must be a real number in [0.0, 1.0]. Booleans are rejected (even though `isinstance(True, int)`).- """+def _check_one_threshold(v: Any) -> float:if isinstance(v, bool) or not isinstance(v, (int, float)) or not math.isfinite(v) or v < 0.0 or v > 1.0:raise ValueError("min_confidence must be a float in [0.0, 1.0], got %r" % (v,))return float(v)+def check_min_confidence_map(m: Dict[Any, Any]) -> Dict[str, float]:+ """Validate a per-bucket abstention-threshold map (#394).++ Keys are option-count bucket strings in `common.temp_bucket`'s spelling -- ``"choice:2"``,+ ``"choice:3-5"``, ``"score:6-10"``, ``"noul:2"`` and so on -- plus an optional ``"default"``+ used for any bucket the map does not name. Values are floats in [0.0, 1.0]. One confidence+ threshold does not transfer across option counts (#394); this lets a caller gate each bucket+ at the level its calibration actually earns. Fit one with+ :func:`laya.calibrate.fit_abstention_thresholds`.+ """+ if not isinstance(m, dict) or not m:+ raise ValueError("a min_confidence map must be a non-empty dict of bucket -> float, got %r" % (m,))+ out: Dict[str, float] = {}+ for key, val in m.items():+ if not isinstance(key, str):+ raise ValueError("min_confidence map keys must be strings like 'choice:3-5', got %r" % (key,))+ out[key] = _check_one_threshold(val)+ return out+++def check_min_confidence(v: Any):+ """Validate opt-in abstention threshold `min_confidence` (#361, #394).++ Either a real number in [0.0, 1.0] (one threshold for every answer; booleans rejected even+ though `isinstance(True, int)`), or a per-bucket mapping (see :func:`check_min_confidence_map`)+ so the threshold can differ by option count. Returns the value in its validated form -- a+ `float` for the scalar case, a `dict[str, float]` for the mapping case -- which the gate+ functions below both accept.+ """+ if isinstance(v, dict):+ return check_min_confidence_map(v)+ return _check_one_threshold(v)+++# Bucket spelling mirrors `common.temp_bucket` but is reproduced here so this module stays+# torch-free (it must import without PyTorch). The answer already carries its type name and, via+# `probabilities`, its option count, so no checkpoint config is needed.+def _option_bucket(answer: Dict[str, Any]) -> Optional[str]:+ qt = answer.get("type")+ if qt not in ("choice", "score", "noul"):+ return None+ probs = answer.get("probabilities")+ if isinstance(probs, dict) and probs:+ k = len(probs)+ elif qt == "noul":+ k = 2+ else:+ return None+ size = "2" if k <= 2 else "3-5" if k <= 5 else "6-10" if k <= 10 else "11+"+ return "%s:%s" % (qt, size)+++def resolve_min_confidence(answer: Dict[str, Any], thresholds: Dict[str, float],+ default: float = 0.0) -> float:+ """The threshold this answer's option-count bucket is gated at, under a per-bucket map.++ Falls back to the map's ``"default"`` entry, then to `default` (0.0 -- gate nothing), for a+ bucket the map does not name, so an unconfigured bucket never abstains by surprise.+ """+ key = _option_bucket(answer)+ if key is not None and key in thresholds:+ return thresholds[key]+ return thresholds.get("default", default)++def flag_low_confidence(results: List[Dict[str, Any]], min_confidence: float) -> None:"""Opt-in abstention marker (#361): flag answers whose confidence falls below `min_confidence`.@@ -74,8 +135,13 @@ def flag_low_confidence(results: List[Dict[str, Any]], min_confidence: float) ->that does not drift with the number of options), falling back to `confidence` if`answer_confidence` is absent.The raw answer and confidence stay intact; `low_confidence: True` is added.++ `min_confidence` is either a float (one threshold for every answer) or a per-bucket mapping+ (#394), in which case each answer is gated at the threshold of its own option-count bucket via+ :func:`resolve_min_confidence`."""- if min_confidence == 0.0:+ is_map = isinstance(min_confidence, dict)+ if not is_map and min_confidence == 0.0:returnfor res in results:answers = res.get("answers") if isinstance(res, dict) else None@@ -85,7 +151,10 @@ def flag_low_confidence(results: List[Dict[str, Any]], min_confidence: float) ->if not isinstance(a, dict):continueconf = _gate_confidence(a)- if conf is not None and conf < min_confidence:+ if conf is None:+ continue+ thr = resolve_min_confidence(a, min_confidence) if is_map else min_confidence+ if conf < thr:a["low_confidence"] = True@@ -136,6 +205,7 @@ def apply_confidence_gate(results: List[Dict[str, Any]], min_confidence: Optiona"""if min_confidence is None:return+ is_map = isinstance(min_confidence, dict)flag_low_confidence(results, min_confidence)for res in results:answers = res.get("answers") if isinstance(res, dict) else None@@ -150,4 +220,6 @@ def apply_confidence_gate(results: List[Dict[str, Any]], min_confidence: Optionaa["abstention"] = GATE_UNEVALUATEDelse:a["abstention"] = GATE_PASSED- a["abstention_threshold"] = float(min_confidence)+ # With a per-bucket map, echo the threshold this answer's bucket was actually gated at.+ a["abstention_threshold"] = float(+ resolve_min_confidence(a, min_confidence) if is_map else min_confidence)diff --git a/tests/test_conformal_abstention.py b/tests/test_conformal_abstention.pynew file mode 100644index 00000000..cd2d0eaa--- /dev/null+++ b/tests/test_conformal_abstention.py@@ -0,0 +1,150 @@+"""Per-bucket abstention thresholds (#394): fit one cut per option-count bucket so a single+confidence threshold's failure to transfer across option counts is fixed.++Weight-free: synthetic (qtype, logits, target, k) records with deliberately different confidence+regimes per bucket, and synthetic answer dicts for the gate. No model, no download.++Run: python tests/test_conformal_abstention.py+"""+import math+import os+import sys++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from laya.calibrate import fit_abstention_thresholds # noqa: E402+from laya.confidence import ( # noqa: E402+ check_min_confidence,+ check_min_confidence_map,+ resolve_min_confidence,+ flag_low_confidence,+ apply_confidence_gate,+ GATE_ABSTAINED,+ GATE_PASSED,+)++PASS, FAIL = [], []+++def check(name, got, want):+ (PASS if got == want else FAIL).append(name if got == want else "%s: got %r want %r" % (name, got, want))+++def check_true(name, cond, detail=""):+ (PASS if cond else FAIL).append(name if cond else "%s %s" % (name, detail))+++def check_raises(name, exc, fn):+ try:+ fn()+ except exc:+ PASS.append(name)+ except Exception as e: # noqa: BLE001+ FAIL.append("%s: raised %r not %s" % (name, e, exc.__name__))+ else:+ FAIL.append("%s: did not raise %s" % (name, exc.__name__))+++# ---- build a (qtype, logits, target, k) record realising a given (k, confidence, correct) ----+def record(k, conf, correct, qtype=0):+ conf = min(max(conf, 1e-3), 1 - 1e-3)+ L = math.log(conf * (k - 1) / (1 - conf)) # logit on index 0 so softmax(z).max() == conf+ z = [0.0] * k+ z[0] = L+ target = [0.0] * k+ target[0 if correct else 1] = 1.0 # argmax(z)==0; correct iff gold is 0+ return (qtype, z, target, k)+++# Two choice buckets with DIFFERENT confidence regimes:+# k=2 : overconfident -- a 0.85-confidence cohort is only half right, so a low cut lets errors in.+# k=12 : well separated -- a 0.50-confidence cohort is already right, so a low cut is safe.+records = []+for _ in range(120):+ records.append(record(2, 0.97, True)) # k=2 confident & correct+for i in range(120):+ records.append(record(2, 0.85, i % 2 == 0)) # k=2 confident but 50% wrong+for _ in range(120):+ records.append(record(12, 0.55, True)) # k=12 modest conf, correct+for i in range(120):+ records.append(record(12, 0.33, i % 5 == 0)) # k=12 low conf, 80% wrong++TARGET = 0.10+thr = fit_abstention_thresholds(records, [1.0, 1.0, 1.0], {}, target_error=TARGET, min_bucket_n=50)+check_true("fit/keyed by temp_bucket", set(thr) == {"choice:2", "choice:11+"}, thr)+check_true("fit/k=2 needs a high cut", thr["choice:2"] > 0.9, thr)+check_true("fit/k=12 accepts a lower cut", thr["choice:11+"] < 0.6, thr)+check_true("fit/the two cuts genuinely differ", thr["choice:2"] - thr["choice:11+"] > 0.3, thr)+++# ---- apply per-bucket vs one global cut, measure error among accepted per bucket ----+def err_and_cov(records, cut_fn):+ out = {}+ for qt, z, t, k in records:+ import numpy as np+ p = np.exp(np.array(z) - max(z)); p = p / p.sum()+ conf = float(p.max()); correct = int(p.argmax()) == int(np.argmax(t))+ key = "choice:2" if k <= 2 else "choice:11+"+ if conf >= cut_fn(key):+ a, e = out.get(key, (0, 0))+ out[key] = (a + 1, e + (0 if correct else 1))+ return out++per_bucket = err_and_cov(records, lambda key: thr[key])+for key, (acc, err) in per_bucket.items():+ rate = err / acc if acc else 0.0+ check_true("apply/per-bucket holds target in %s (err %.0f%%, n=%d)" % (key, rate * 100, acc),+ acc > 0 and rate <= TARGET + 0.02, (acc, err))++# one global cut = the per-bucket cut for k=2 (0.9+); it over-abstains k=12 to near-zero coverage,+# which is exactly the #394 failure: a cut tuned for one option count does not serve the other.+GLOBAL = thr["choice:2"]+glob = err_and_cov(records, lambda key: GLOBAL)+k12_acc = glob.get("choice:11+", (0, 0))[0]+check_true("transfer/global k=2 cut starves k=12 coverage (#394)", k12_acc == 0, glob)+# and the mirror: the k=12 cut lets k=2 errors through+GLOBAL2 = thr["choice:11+"]+glob2 = err_and_cov(records, lambda key: GLOBAL2)+k2_acc, k2_err = glob2.get("choice:2", (0, 0))+check_true("transfer/global k=12 cut exceeds target error on k=2 (#394)",+ k2_acc > 0 and (k2_err / k2_acc) > TARGET, (k2_acc, k2_err))+++# ---- confidence.py: resolve + gate with a map ----+def ans(qtype, k, conf):+ probs = {str(i): 0.0 for i in range(k)}+ return {"type": qtype, "probabilities": probs, "answer_confidence": conf}++MAP = {"choice:2": 0.9, "choice:11+": 0.5, "noul:2": 0.8, "default": 0.3}+check("resolve/choice k=2 bucket", resolve_min_confidence(ans("choice", 2, 0.5), MAP), 0.9)+check("resolve/choice k=12 bucket (11+)", resolve_min_confidence(ans("choice", 12, 0.5), MAP), 0.5)+check("resolve/choice k=4 falls to default", resolve_min_confidence(ans("choice", 4, 0.5), MAP), 0.3)+check("resolve/noul bucket", resolve_min_confidence({"type": "noul", "answer_confidence": 0.5}, MAP), 0.8)+check("resolve/unknown + no default -> 0.0 (gate nothing)",+ resolve_min_confidence(ans("score", 7, 0.5), {"choice:2": 0.9}), 0.0)++# gate a batch with a per-bucket map: a 0.85-confident k=2 answer abstains (cut 0.9), a 0.6-confident+# k=12 answer passes (cut 0.5) -- a single float could not do both.+results = [{"answers": {"a": ans("choice", 2, 0.85), "b": ans("choice", 12, 0.60)}}]+apply_confidence_gate(results, MAP)+check("gate/k=2 low-conf abstains under its bucket cut", results[0]["answers"]["a"]["abstention"], GATE_ABSTAINED)+check("gate/k=12 passes under its lower bucket cut", results[0]["answers"]["b"]["abstention"], GATE_PASSED)+check("gate/echoes the per-bucket threshold (k=2)", results[0]["answers"]["a"]["abstention_threshold"], 0.9)+check("gate/echoes the per-bucket threshold (k=12)", results[0]["answers"]["b"]["abstention_threshold"], 0.5)++# the float path is unchanged+res_f = [{"answers": {"a": ans("choice", 3, 0.4)}}]+flag_low_confidence(res_f, 0.7)+check("float/below a scalar threshold still flags", res_f[0]["answers"]["a"].get("low_confidence"), True)+check("validate/float still accepted", check_min_confidence(0.7), 0.7)+check("validate/map accepted and normalised", check_min_confidence({"choice:2": 0.9}), {"choice:2": 0.9})+check_raises("validate/map value out of range rejected", ValueError,+ lambda: check_min_confidence_map({"choice:2": 1.5}))+check_raises("validate/map non-string key rejected", ValueError,+ lambda: check_min_confidence_map({2: 0.9}))+++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))+for f in FAIL:+ print(" FAIL " + f)+sys.exit(1 if FAIL else 0)