Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(confidence): per-option-count abstention thresholds (Closes #394)

AlKor133d762c4fdb75
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c97f733d..a7da8150 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -149,6 +149,7 @@ jobs:
python tests/test_shortlist_cosine.py
python tests/test_temperature_loading.py
python tests/test_confidence.py
+ python tests/test_conformal_abstention.py
python tests/test_cli.py
python tests/test_cli_lang_guess.py
python tests/test_mcp.py
diff --git a/docs/reference/helpers.md b/docs/reference/helpers.md
index eeda50a0..bdedf8be 100644
--- a/docs/reference/helpers.md
+++ b/docs/reference/helpers.md
@@ -42,6 +42,10 @@
::: laya.confidence.check_min_confidence
+::: laya.confidence.check_min_confidence_map
+
+::: laya.confidence.resolve_min_confidence
+
::: laya.confidence.flag_low_confidence
::: laya.confidence.apply_confidence_gate
@@ -62,6 +66,8 @@
::: laya.calibrate.fit_temperature_map
+::: laya.calibrate.fit_abstention_thresholds
+
::: laya.common.render_options
::: laya.common.proper_reward
diff --git a/laya/__init__.py b/laya/__init__.py
index f1ffe8a8..a3ac90d6 100644
--- a/laya/__init__.py
+++ b/laya/__init__.py
@@ -28,12 +28,15 @@ _LAZY_ATTRS = {
"fit_temperatures": (".calibrate", "fit_temperatures"),
"fit_one_temperature": (".calibrate", "fit_one_temperature"),
"fit_temperature_map": (".calibrate", "fit_temperature_map"),
+ "fit_abstention_thresholds": (".calibrate", "fit_abstention_thresholds"),
"proper_reward": (".common", "proper_reward"),
"td_lambda_targets": (".common", "td_lambda_targets"),
"ece_score": (".common", "ece_score"),
"answer_confidence": (".common", "answer_confidence"),
"confidence_from_probs": (".common", "confidence_from_probs"),
"check_min_confidence": (".confidence", "check_min_confidence"),
+ "check_min_confidence_map": (".confidence", "check_min_confidence_map"),
+ "resolve_min_confidence": (".confidence", "resolve_min_confidence"),
"flag_low_confidence": (".confidence", "flag_low_confidence"),
"apply_confidence_gate": (".confidence", "apply_confidence_gate"),
"GATE_STATES": (".confidence", "GATE_STATES"),
@@ -76,6 +79,7 @@ __all__ = [
"fit_temperatures",
"fit_one_temperature",
"fit_temperature_map",
+ "fit_abstention_thresholds",
"Router",
"RouteDecision",
"DEFAULT_MODELS",
@@ -99,6 +103,8 @@ __all__ = [
"answer_confidence",
"confidence_from_probs",
"check_min_confidence",
+ "check_min_confidence_map",
+ "resolve_min_confidence",
"flag_low_confidence",
"apply_confidence_gate",
"GATE_STATES",
diff --git a/laya/calibrate.py b/laya/calibrate.py
index 3906a5fe..e57a7c45 100644
--- a/laya/calibrate.py
+++ b/laya/calibrate.py
@@ -258,6 +258,84 @@ def fit_temperature_map(records: Iterable, compute_ece: bool = False, seed: int
fit_temperatures = fit_temperature_map
+# Per-bucket floor for abstention thresholds. Lower than `MIN_BUCKET_N` (which governs temperature
+# fitting): a threshold is a single order statistic of the calibrated confidences, so it stabilises
+# on far fewer examples than an LBFGS temperature does. Buckets below this are omitted; the caller's
+# scalar `min_confidence` (or the map's "default") covers them.
+MIN_ABSTAIN_BUCKET_N = 100
+
+
+def _select_abstention_threshold(pairs: Sequence[Tuple[float, int]], target_error: float,
+ conservative: bool) -> float:
+ """Smallest confidence `tau` whose accepted set (`conf >= tau`) keeps error <= `target_error`.
+
+ `pairs` are `(confidence, correct)`. Sweeping from the most confident down maximises coverage
+ at the target risk (the selective-classification / split-conformal cut). `conservative` adds one
+ pseudo-error so a bucket does not clear the gate on a lucky short run. Returns 1.0 when no cut
+ holds the risk -- the bucket is too unreliable to accept anything short of a reported certainty.
+ """
+ ordered = sorted(pairs, key=lambda cc: cc[0], reverse=True)
+ levels = sorted({c for c, _ in ordered}, reverse=True)
+ n = len(ordered)
+ n_acc = n_err = idx = 0
+ best = None
+ # Evaluate the error over the WHOLE accepted set {conf >= tau} at each distinct level, not
+ # incrementally within a tie: a threshold accepts every answer at its own confidence, so a tie
+ # group's errors must all be counted before the level is judged (else the cut sinks into a bad
+ # cohort on its first few correct members).
+ for tau in levels:
+ while idx < n and ordered[idx][0] >= tau:
+ n_acc += 1
+ n_err += 0 if ordered[idx][1] else 1
+ idx += 1
+ rate = (n_err + 1.0) / (n_acc + 1.0) if conservative else (n_err / n_acc)
+ if n_acc > 0 and rate <= target_error:
+ best = tau
+ if best is None:
+ return 1.0
+ return float(min(1.0, max(0.0, best)))
+
+
+def fit_abstention_thresholds(records: Iterable, temperature: Sequence[float],
+ temperature_by_options: Dict[str, float], *,
+ target_error: float = 0.10,
+ min_bucket_n: int = MIN_ABSTAIN_BUCKET_N,
+ conservative: bool = True) -> Dict[str, float]:
+ """Fit a per-`temp_bucket` abstention threshold so a gate keeps a target error in every bucket.
+
+ A single `min_confidence` does not transfer across option counts (#394): the calibrated
+ confidence of a 2-option and a 12-option answer live on different scales, so one cut over- or
+ under-abstains depending on the question. This fits one cut per bucket instead, keyed exactly
+ like `temperature_by_options` (`common.temp_bucket`, e.g. ``"choice:3-5"``), and the result is a
+ `min_confidence` map that :func:`laya.confidence.check_min_confidence` /
+ :func:`laya.confidence.apply_confidence_gate` accept directly.
+
+ `records` are the same `(qtype, logits, target[, k])` tuples `fit_temperature_map` consumes
+ (`records_from_labeled` builds them). Confidence is the **calibrated** `max(p)` -- the logits are
+ scaled by the fitted `temperature` / `temperature_by_options` first, so thresholds and the
+ numbers the runtime reports are on the same scale. `target_error` is the tolerated error among
+ accepted answers; `min_bucket_n` omits buckets too small to fit, and `conservative` adds a
+ one-sample margin. The thresholds are empirical cuts on the calibration set, not a formal
+ coverage guarantee -- validate on held-out data (`fit_temperature_map(..., compute_ece=True)`
+ gives a held-out split) for a production gate.
+ """
+ if not 0.0 <= target_error <= 1.0:
+ raise ValueError("target_error must be in [0.0, 1.0], got %r" % (target_error,))
+ recs = _iter_records(records)
+ by_bucket: Dict[str, List[Tuple[float, int]]] = {}
+ for qt, z, t, k in recs:
+ y = int(np.argmax(t[:k]))
+ t_scale = temperature_by_options.get(temp_bucket(qt, k), temperature[qt])
+ p = _softmax(z[:k], t_scale)
+ by_bucket.setdefault(temp_bucket(qt, k), []).append((float(p.max()), int(int(p.argmax()) == y)))
+ out: Dict[str, float] = {}
+ for key, pairs in by_bucket.items():
+ if len(pairs) < min_bucket_n:
+ continue
+ out[key] = _select_abstention_threshold(pairs, target_error, conservative)
+ return out
+
+
def records_from_labeled(agent, pairs: Sequence) -> List[Record]:
"""Collect CPU records from `(state, questions, targets)`.
diff --git a/laya/confidence.py b/laya/confidence.py
index 43b67ee5..57401b3c 100644
--- a/laya/confidence.py
+++ b/laya/confidence.py
@@ -57,16 +57,77 @@ def _gate_confidence(answer: Dict[str, Any]) -> Optional[float]:
return None
-def check_min_confidence(v: Any) -> float:
- """Validate opt-in abstention threshold `min_confidence` (#361).
-
- Must be a real number in [0.0, 1.0]. Booleans are rejected (even though `isinstance(True, int)`).
- """
+def _check_one_threshold(v: Any) -> float:
if isinstance(v, bool) or not isinstance(v, (int, float)) or not math.isfinite(v) or v < 0.0 or v > 1.0:
raise ValueError("min_confidence must be a float in [0.0, 1.0], got %r" % (v,))
return float(v)
+def check_min_confidence_map(m: Dict[Any, Any]) -> Dict[str, float]:
+ """Validate a per-bucket abstention-threshold map (#394).
+
+ Keys are option-count bucket strings in `common.temp_bucket`'s spelling -- ``"choice:2"``,
+ ``"choice:3-5"``, ``"score:6-10"``, ``"noul:2"`` and so on -- plus an optional ``"default"``
+ used for any bucket the map does not name. Values are floats in [0.0, 1.0]. One confidence
+ threshold does not transfer across option counts (#394); this lets a caller gate each bucket
+ at the level its calibration actually earns. Fit one with
+ :func:`laya.calibrate.fit_abstention_thresholds`.
+ """
+ if not isinstance(m, dict) or not m:
+ raise ValueError("a min_confidence map must be a non-empty dict of bucket -> float, got %r" % (m,))
+ out: Dict[str, float] = {}
+ for key, val in m.items():
+ if not isinstance(key, str):
+ raise ValueError("min_confidence map keys must be strings like 'choice:3-5', got %r" % (key,))
+ out[key] = _check_one_threshold(val)
+ return out
+
+
+def check_min_confidence(v: Any):
+ """Validate opt-in abstention threshold `min_confidence` (#361, #394).
+
+ Either a real number in [0.0, 1.0] (one threshold for every answer; booleans rejected even
+ though `isinstance(True, int)`), or a per-bucket mapping (see :func:`check_min_confidence_map`)
+ so the threshold can differ by option count. Returns the value in its validated form -- a
+ `float` for the scalar case, a `dict[str, float]` for the mapping case -- which the gate
+ functions below both accept.
+ """
+ if isinstance(v, dict):
+ return check_min_confidence_map(v)
+ return _check_one_threshold(v)
+
+
+# Bucket spelling mirrors `common.temp_bucket` but is reproduced here so this module stays
+# torch-free (it must import without PyTorch). The answer already carries its type name and, via
+# `probabilities`, its option count, so no checkpoint config is needed.
+def _option_bucket(answer: Dict[str, Any]) -> Optional[str]:
+ qt = answer.get("type")
+ if qt not in ("choice", "score", "noul"):
+ return None
+ probs = answer.get("probabilities")
+ if isinstance(probs, dict) and probs:
+ k = len(probs)
+ elif qt == "noul":
+ k = 2
+ else:
+ return None
+ size = "2" if k <= 2 else "3-5" if k <= 5 else "6-10" if k <= 10 else "11+"
+ return "%s:%s" % (qt, size)
+
+
+def resolve_min_confidence(answer: Dict[str, Any], thresholds: Dict[str, float],
+ default: float = 0.0) -> float:
+ """The threshold this answer's option-count bucket is gated at, under a per-bucket map.
+
+ Falls back to the map's ``"default"`` entry, then to `default` (0.0 -- gate nothing), for a
+ bucket the map does not name, so an unconfigured bucket never abstains by surprise.
+ """
+ key = _option_bucket(answer)
+ if key is not None and key in thresholds:
+ return thresholds[key]
+ return thresholds.get("default", default)
+
+
def flag_low_confidence(results: List[Dict[str, Any]], min_confidence: float) -> None:
"""Opt-in abstention marker (#361): flag answers whose confidence falls below `min_confidence`.
@@ -74,8 +135,13 @@ def flag_low_confidence(results: List[Dict[str, Any]], min_confidence: float) ->
that does not drift with the number of options), falling back to `confidence` if
`answer_confidence` is absent.
The raw answer and confidence stay intact; `low_confidence: True` is added.
+
+ `min_confidence` is either a float (one threshold for every answer) or a per-bucket mapping
+ (#394), in which case each answer is gated at the threshold of its own option-count bucket via
+ :func:`resolve_min_confidence`.
"""
- if min_confidence == 0.0:
+ is_map = isinstance(min_confidence, dict)
+ if not is_map and min_confidence == 0.0:
return
for res in results:
answers = res.get("answers") if isinstance(res, dict) else None
@@ -85,7 +151,10 @@ def flag_low_confidence(results: List[Dict[str, Any]], min_confidence: float) ->
if not isinstance(a, dict):
continue
conf = _gate_confidence(a)
- if conf is not None and conf < min_confidence:
+ if conf is None:
+ continue
+ thr = resolve_min_confidence(a, min_confidence) if is_map else min_confidence
+ if conf < thr:
a["low_confidence"] = True
@@ -136,6 +205,7 @@ def apply_confidence_gate(results: List[Dict[str, Any]], min_confidence: Optiona
"""
if min_confidence is None:
return
+ is_map = isinstance(min_confidence, dict)
flag_low_confidence(results, min_confidence)
for res in results:
answers = res.get("answers") if isinstance(res, dict) else None
@@ -150,4 +220,6 @@ def apply_confidence_gate(results: List[Dict[str, Any]], min_confidence: Optiona
a["abstention"] = GATE_UNEVALUATED
else:
a["abstention"] = GATE_PASSED
- a["abstention_threshold"] = float(min_confidence)
+ # With a per-bucket map, echo the threshold this answer's bucket was actually gated at.
+ a["abstention_threshold"] = float(
+ resolve_min_confidence(a, min_confidence) if is_map else min_confidence)
diff --git a/tests/test_conformal_abstention.py b/tests/test_conformal_abstention.py
new file mode 100644
index 00000000..cd2d0eaa
--- /dev/null
+++ b/tests/test_conformal_abstention.py
@@ -0,0 +1,150 @@
+"""Per-bucket abstention thresholds (#394): fit one cut per option-count bucket so a single
+confidence threshold's failure to transfer across option counts is fixed.
+
+Weight-free: synthetic (qtype, logits, target, k) records with deliberately different confidence
+regimes per bucket, and synthetic answer dicts for the gate. No model, no download.
+
+Run: python tests/test_conformal_abstention.py
+"""
+import math
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from laya.calibrate import fit_abstention_thresholds # noqa: E402
+from laya.confidence import ( # noqa: E402
+ check_min_confidence,
+ check_min_confidence_map,
+ resolve_min_confidence,
+ flag_low_confidence,
+ apply_confidence_gate,
+ GATE_ABSTAINED,
+ GATE_PASSED,
+)
+
+PASS, FAIL = [], []
+
+
+def check(name, got, want):
+ (PASS if got == want else FAIL).append(name if got == want else "%s: got %r want %r" % (name, got, want))
+
+
+def check_true(name, cond, detail=""):
+ (PASS if cond else FAIL).append(name if cond else "%s %s" % (name, detail))
+
+
+def check_raises(name, exc, fn):
+ try:
+ fn()
+ except exc:
+ PASS.append(name)
+ except Exception as e: # noqa: BLE001
+ FAIL.append("%s: raised %r not %s" % (name, e, exc.__name__))
+ else:
+ FAIL.append("%s: did not raise %s" % (name, exc.__name__))
+
+
+# ---- build a (qtype, logits, target, k) record realising a given (k, confidence, correct) ----
+def record(k, conf, correct, qtype=0):
+ conf = min(max(conf, 1e-3), 1 - 1e-3)
+ L = math.log(conf * (k - 1) / (1 - conf)) # logit on index 0 so softmax(z).max() == conf
+ z = [0.0] * k
+ z[0] = L
+ target = [0.0] * k
+ target[0 if correct else 1] = 1.0 # argmax(z)==0; correct iff gold is 0
+ return (qtype, z, target, k)
+
+
+# Two choice buckets with DIFFERENT confidence regimes:
+# k=2 : overconfident -- a 0.85-confidence cohort is only half right, so a low cut lets errors in.
+# k=12 : well separated -- a 0.50-confidence cohort is already right, so a low cut is safe.
+records = []
+for _ in range(120):
+ records.append(record(2, 0.97, True)) # k=2 confident & correct
+for i in range(120):
+ records.append(record(2, 0.85, i % 2 == 0)) # k=2 confident but 50% wrong
+for _ in range(120):
+ records.append(record(12, 0.55, True)) # k=12 modest conf, correct
+for i in range(120):
+ records.append(record(12, 0.33, i % 5 == 0)) # k=12 low conf, 80% wrong
+
+TARGET = 0.10
+thr = fit_abstention_thresholds(records, [1.0, 1.0, 1.0], {}, target_error=TARGET, min_bucket_n=50)
+check_true("fit/keyed by temp_bucket", set(thr) == {"choice:2", "choice:11+"}, thr)
+check_true("fit/k=2 needs a high cut", thr["choice:2"] > 0.9, thr)
+check_true("fit/k=12 accepts a lower cut", thr["choice:11+"] < 0.6, thr)
+check_true("fit/the two cuts genuinely differ", thr["choice:2"] - thr["choice:11+"] > 0.3, thr)
+
+
+# ---- apply per-bucket vs one global cut, measure error among accepted per bucket ----
+def err_and_cov(records, cut_fn):
+ out = {}
+ for qt, z, t, k in records:
+ import numpy as np
+ p = np.exp(np.array(z) - max(z)); p = p / p.sum()
+ conf = float(p.max()); correct = int(p.argmax()) == int(np.argmax(t))
+ key = "choice:2" if k <= 2 else "choice:11+"
+ if conf >= cut_fn(key):
+ a, e = out.get(key, (0, 0))
+ out[key] = (a + 1, e + (0 if correct else 1))
+ return out
+
+per_bucket = err_and_cov(records, lambda key: thr[key])
+for key, (acc, err) in per_bucket.items():
+ rate = err / acc if acc else 0.0
+ check_true("apply/per-bucket holds target in %s (err %.0f%%, n=%d)" % (key, rate * 100, acc),
+ acc > 0 and rate <= TARGET + 0.02, (acc, err))
+
+# one global cut = the per-bucket cut for k=2 (0.9+); it over-abstains k=12 to near-zero coverage,
+# which is exactly the #394 failure: a cut tuned for one option count does not serve the other.
+GLOBAL = thr["choice:2"]
+glob = err_and_cov(records, lambda key: GLOBAL)
+k12_acc = glob.get("choice:11+", (0, 0))[0]
+check_true("transfer/global k=2 cut starves k=12 coverage (#394)", k12_acc == 0, glob)
+# and the mirror: the k=12 cut lets k=2 errors through
+GLOBAL2 = thr["choice:11+"]
+glob2 = err_and_cov(records, lambda key: GLOBAL2)
+k2_acc, k2_err = glob2.get("choice:2", (0, 0))
+check_true("transfer/global k=12 cut exceeds target error on k=2 (#394)",
+ k2_acc > 0 and (k2_err / k2_acc) > TARGET, (k2_acc, k2_err))
+
+
+# ---- confidence.py: resolve + gate with a map ----
+def ans(qtype, k, conf):
+ probs = {str(i): 0.0 for i in range(k)}
+ return {"type": qtype, "probabilities": probs, "answer_confidence": conf}
+
+MAP = {"choice:2": 0.9, "choice:11+": 0.5, "noul:2": 0.8, "default": 0.3}
+check("resolve/choice k=2 bucket", resolve_min_confidence(ans("choice", 2, 0.5), MAP), 0.9)
+check("resolve/choice k=12 bucket (11+)", resolve_min_confidence(ans("choice", 12, 0.5), MAP), 0.5)
+check("resolve/choice k=4 falls to default", resolve_min_confidence(ans("choice", 4, 0.5), MAP), 0.3)
+check("resolve/noul bucket", resolve_min_confidence({"type": "noul", "answer_confidence": 0.5}, MAP), 0.8)
+check("resolve/unknown + no default -> 0.0 (gate nothing)",
+ resolve_min_confidence(ans("score", 7, 0.5), {"choice:2": 0.9}), 0.0)
+
+# gate a batch with a per-bucket map: a 0.85-confident k=2 answer abstains (cut 0.9), a 0.6-confident
+# k=12 answer passes (cut 0.5) -- a single float could not do both.
+results = [{"answers": {"a": ans("choice", 2, 0.85), "b": ans("choice", 12, 0.60)}}]
+apply_confidence_gate(results, MAP)
+check("gate/k=2 low-conf abstains under its bucket cut", results[0]["answers"]["a"]["abstention"], GATE_ABSTAINED)
+check("gate/k=12 passes under its lower bucket cut", results[0]["answers"]["b"]["abstention"], GATE_PASSED)
+check("gate/echoes the per-bucket threshold (k=2)", results[0]["answers"]["a"]["abstention_threshold"], 0.9)
+check("gate/echoes the per-bucket threshold (k=12)", results[0]["answers"]["b"]["abstention_threshold"], 0.5)
+
+# the float path is unchanged
+res_f = [{"answers": {"a": ans("choice", 3, 0.4)}}]
+flag_low_confidence(res_f, 0.7)
+check("float/below a scalar threshold still flags", res_f[0]["answers"]["a"].get("low_confidence"), True)
+check("validate/float still accepted", check_min_confidence(0.7), 0.7)
+check("validate/map accepted and normalised", check_min_confidence({"choice:2": 0.9}), {"choice:2": 0.9})
+check_raises("validate/map value out of range rejected", ValueError,
+ lambda: check_min_confidence_map({"choice:2": 1.5}))
+check_raises("validate/map non-string key rejected", ValueError,
+ lambda: check_min_confidence_map({2: 0.9}))
+
+
+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
+for f in FAIL:
+ print(" FAIL " + f)
+sys.exit(1 if FAIL else 0)