Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(calibrate): fit abstention thresholds on the scale the gate will read

AlKor1309033d10a8cf
diff --git a/laya/calibrate.py b/laya/calibrate.py
index d25ce6a9..868d7b03 100644
--- a/laya/calibrate.py
+++ b/laya/calibrate.py
@@ -337,6 +337,7 @@ def _select_abstention_threshold(pairs: Sequence[Tuple[float, int]], target_erro
def fit_abstention_thresholds(records: Iterable, temperature: Sequence[float],
temperature_by_options: Dict[str, float], *,
+ binning_map: Optional[Dict[str, Dict[str, Any]]] = None,
target_error: float = 0.10,
min_bucket_n: int = MIN_ABSTAIN_BUCKET_N,
conservative: bool = True) -> Dict[str, float]:
@@ -357,6 +358,15 @@ def fit_abstention_thresholds(records: Iterable, temperature: Sequence[float],
one-sample margin. The thresholds are empirical cuts on the calibration set, not a formal
coverage guarantee -- validate on held-out data (`fit_temperature_map(..., compute_ece=True)`
gives a held-out split) for a production gate.
+
+ Pass `binning_map` when the agent that will serve these thresholds has one installed -- by
+ `Agent.fit_binning`, or by a calibration payload that carries `binning_map` -- because the
+ runtime recalibrates `answer_confidence` through that map before anything reads it, so a cut
+ fitted without it is a cut on a scale the gate never sees. The thresholds are then on the binned
+ scale, and the order the two were fitted in stops mattering. Measured on 1,200 synthetic
+ 12-option records at `target_error=0.10`: the cut fitted without a map holds 9.8% error over 50%
+ coverage on un-binned confidences, and admits 94.5% of answers at 25.6% error once the same
+ number is compared against binned ones.
"""
if not 0.0 <= target_error <= 1.0:
raise ValueError("target_error must be in [0.0, 1.0], got %r" % (target_error,))
@@ -364,9 +374,15 @@ def fit_abstention_thresholds(records: Iterable, temperature: Sequence[float],
by_bucket: Dict[str, List[Tuple[float, int]]] = {}
for qt, z, t, k in recs:
y = int(np.argmax(t[:k]))
- t_scale = temperature_by_options.get(temp_bucket(qt, k), temperature[qt])
+ bucket = temp_bucket(qt, k)
+ t_scale = temperature_by_options.get(bucket, temperature[qt])
p = _softmax(z[:k], t_scale)
- by_bucket.setdefault(temp_bucket(qt, k), []).append((float(p.max()), int(int(p.argmax()) == y)))
+ conf = float(p.max())
+ if binning_map:
+ # The runtime bins before anyone reads `answer_confidence` (`Agent._decode_answers`),
+ # so the cut has to be chosen on the binned scale or it gates a different quantity.
+ conf = apply_binning_map(conf, bucket, binning_map)
+ by_bucket.setdefault(bucket, []).append((conf, int(int(p.argmax()) == y)))
out: Dict[str, float] = {}
for key, pairs in by_bucket.items():
if len(pairs) < min_bucket_n:
@@ -407,9 +423,10 @@ def fit_binning_map(records: Iterable, temperature: Sequence[float],
by_bucket: Dict[str, List[Tuple[float, int]]] = {}
for qt, z, t, k in recs:
y = int(np.argmax(t[:k]))
- t_scale = temperature_by_options.get(temp_bucket(qt, k), temperature[qt])
+ bucket = temp_bucket(qt, k)
+ t_scale = temperature_by_options.get(bucket, temperature[qt])
p = _softmax(z[:k], t_scale)
- by_bucket.setdefault(temp_bucket(qt, k), []).append((float(p.max()), int(int(p.argmax()) == y)))
+ by_bucket.setdefault(bucket, []).append((float(p.max()), int(int(p.argmax()) == y)))
out: Dict[str, Dict[str, Any]] = {}
for key, pairs in by_bucket.items():
if len(pairs) < min_bucket_n:
diff --git a/tests/test_conformal_abstention.py b/tests/test_conformal_abstention.py
index cd2d0eaa..3d4b8ed2 100644
--- a/tests/test_conformal_abstention.py
+++ b/tests/test_conformal_abstention.py
@@ -12,7 +12,11 @@ import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-from laya.calibrate import fit_abstention_thresholds # noqa: E402
+from laya.calibrate import ( # noqa: E402
+ apply_binning_map,
+ fit_abstention_thresholds,
+ fit_binning_map,
+)
from laya.confidence import ( # noqa: E402
check_min_confidence,
check_min_confidence_map,
@@ -144,6 +148,61 @@ check_raises("validate/map non-string key rejected", ValueError,
lambda: check_min_confidence_map({2: 0.9}))
+
+# --------------------------------------------------------------- thresholds vs a binning map
+# A histogram-binning map recalibrates `answer_confidence` at runtime (`Agent._decode_answers`),
+# so a cut fitted on the temperature-scaled scale gates a quantity the runtime no longer reports:
+# the same number admits far more than its target error. Fitting with the map puts both on one
+# scale, and then the order the two were fitted in stops mattering.
+import numpy as np # noqa: E402
+
+_rng = np.random.default_rng(7)
+_binning_records = []
+for _ in range(1200):
+ _true = int(_rng.integers(12))
+ _z = _rng.normal(0.0, 1.0, 12)
+ _z[_true] += 2.2 # usually right, sometimes not: a continuous spread
+ _t = [0.0] * 12
+ _t[_true] = 1.0
+ _binning_records.append((0, _z.tolist(), _t, 12))
+
+_temps, _tbo = [1.0, 1.0, 1.0], {}
+_bmap = fit_binning_map(_binning_records, _temps, _tbo)
+_cut_raw = fit_abstention_thresholds(_binning_records, _temps, _tbo, target_error=0.10)["choice:11+"]
+_cut_binned = fit_abstention_thresholds(_binning_records, _temps, _tbo, binning_map=_bmap,
+ target_error=0.10)["choice:11+"]
+
+
+def _gate_error(cut, binned):
+ """Coverage and error among accepted, when the runtime reports binned (or raw) confidences."""
+ accepted = wrong = 0
+ for qt, z, t, k in _binning_records:
+ p = np.exp(np.asarray(z) - max(z))
+ p = p / p.sum()
+ conf = float(p.max())
+ if binned:
+ conf = apply_binning_map(conf, "choice:11+", _bmap)
+ if conf >= cut:
+ accepted += 1
+ wrong += int(int(p.argmax()) != int(np.argmax(t)))
+ return accepted, (wrong / accepted if accepted else 0.0)
+
+
+_acc_raw, _err_raw = _gate_error(_cut_raw, binned=False)
+_acc_mismatch, _err_mismatch = _gate_error(_cut_raw, binned=True)
+_acc_fit, _err_fit = _gate_error(_cut_binned, binned=True)
+
+check_true("binning/an un-binned cut holds its target on its own scale", _err_raw <= 0.12,
+ "cut=%.4f accepted=%d error=%.3f" % (_cut_raw, _acc_raw, _err_raw))
+check_true("binning/the same cut stops gating once the runtime bins", _err_mismatch > 0.2,
+ "cut=%.4f accepted=%d error=%.3f" % (_cut_raw, _acc_mismatch, _err_mismatch))
+check_true("binning/fitting with the map restores the target", _err_fit <= 0.12,
+ "cut=%.4f accepted=%d error=%.3f" % (_cut_binned, _acc_fit, _err_fit))
+check_true("binning/the two cuts genuinely differ", abs(_cut_binned - _cut_raw) > 1e-9,
+ "raw=%.4f binned=%.4f" % (_cut_raw, _cut_binned))
+check("binning/no map leaves the fit unchanged",
+ fit_abstention_thresholds(_binning_records, _temps, _tbo, binning_map=None,
+ target_error=0.10)["choice:11+"], _cut_raw)
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL " + f)