Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(calibrate): fit abstention thresholds on the scale the gate will read
AlKor1309033d10a8cf
diff --git a/laya/calibrate.py b/laya/calibrate.pyindex d25ce6a9..868d7b03 100644--- a/laya/calibrate.py+++ b/laya/calibrate.py@@ -337,6 +337,7 @@ def _select_abstention_threshold(pairs: Sequence[Tuple[float, int]], target_errodef fit_abstention_thresholds(records: Iterable, temperature: Sequence[float],temperature_by_options: Dict[str, float], *,+ binning_map: Optional[Dict[str, Dict[str, Any]]] = None,target_error: float = 0.10,min_bucket_n: int = MIN_ABSTAIN_BUCKET_N,conservative: bool = True) -> Dict[str, float]:@@ -357,6 +358,15 @@ def fit_abstention_thresholds(records: Iterable, temperature: Sequence[float],one-sample margin. The thresholds are empirical cuts on the calibration set, not a formalcoverage guarantee -- validate on held-out data (`fit_temperature_map(..., compute_ece=True)`gives a held-out split) for a production gate.++ Pass `binning_map` when the agent that will serve these thresholds has one installed -- by+ `Agent.fit_binning`, or by a calibration payload that carries `binning_map` -- because the+ runtime recalibrates `answer_confidence` through that map before anything reads it, so a cut+ fitted without it is a cut on a scale the gate never sees. The thresholds are then on the binned+ scale, and the order the two were fitted in stops mattering. Measured on 1,200 synthetic+ 12-option records at `target_error=0.10`: the cut fitted without a map holds 9.8% error over 50%+ coverage on un-binned confidences, and admits 94.5% of answers at 25.6% error once the same+ number is compared against binned ones."""if not 0.0 <= target_error <= 1.0:raise ValueError("target_error must be in [0.0, 1.0], got %r" % (target_error,))@@ -364,9 +374,15 @@ def fit_abstention_thresholds(records: Iterable, temperature: Sequence[float],by_bucket: Dict[str, List[Tuple[float, int]]] = {}for qt, z, t, k in recs:y = int(np.argmax(t[:k]))- t_scale = temperature_by_options.get(temp_bucket(qt, k), temperature[qt])+ bucket = temp_bucket(qt, k)+ t_scale = temperature_by_options.get(bucket, temperature[qt])p = _softmax(z[:k], t_scale)- by_bucket.setdefault(temp_bucket(qt, k), []).append((float(p.max()), int(int(p.argmax()) == y)))+ conf = float(p.max())+ if binning_map:+ # The runtime bins before anyone reads `answer_confidence` (`Agent._decode_answers`),+ # so the cut has to be chosen on the binned scale or it gates a different quantity.+ conf = apply_binning_map(conf, bucket, binning_map)+ by_bucket.setdefault(bucket, []).append((conf, int(int(p.argmax()) == y)))out: Dict[str, float] = {}for key, pairs in by_bucket.items():if len(pairs) < min_bucket_n:@@ -407,9 +423,10 @@ def fit_binning_map(records: Iterable, temperature: Sequence[float],by_bucket: Dict[str, List[Tuple[float, int]]] = {}for qt, z, t, k in recs:y = int(np.argmax(t[:k]))- t_scale = temperature_by_options.get(temp_bucket(qt, k), temperature[qt])+ bucket = temp_bucket(qt, k)+ t_scale = temperature_by_options.get(bucket, temperature[qt])p = _softmax(z[:k], t_scale)- by_bucket.setdefault(temp_bucket(qt, k), []).append((float(p.max()), int(int(p.argmax()) == y)))+ by_bucket.setdefault(bucket, []).append((float(p.max()), int(int(p.argmax()) == y)))out: Dict[str, Dict[str, Any]] = {}for key, pairs in by_bucket.items():if len(pairs) < min_bucket_n:diff --git a/tests/test_conformal_abstention.py b/tests/test_conformal_abstention.pyindex cd2d0eaa..3d4b8ed2 100644--- a/tests/test_conformal_abstention.py+++ b/tests/test_conformal_abstention.py@@ -12,7 +12,11 @@ import syssys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))-from laya.calibrate import fit_abstention_thresholds # noqa: E402+from laya.calibrate import ( # noqa: E402+ apply_binning_map,+ fit_abstention_thresholds,+ fit_binning_map,+)from laya.confidence import ( # noqa: E402check_min_confidence,check_min_confidence_map,@@ -144,6 +148,61 @@ check_raises("validate/map non-string key rejected", ValueError,lambda: check_min_confidence_map({2: 0.9}))++# --------------------------------------------------------------- thresholds vs a binning map+# A histogram-binning map recalibrates `answer_confidence` at runtime (`Agent._decode_answers`),+# so a cut fitted on the temperature-scaled scale gates a quantity the runtime no longer reports:+# the same number admits far more than its target error. Fitting with the map puts both on one+# scale, and then the order the two were fitted in stops mattering.+import numpy as np # noqa: E402++_rng = np.random.default_rng(7)+_binning_records = []+for _ in range(1200):+ _true = int(_rng.integers(12))+ _z = _rng.normal(0.0, 1.0, 12)+ _z[_true] += 2.2 # usually right, sometimes not: a continuous spread+ _t = [0.0] * 12+ _t[_true] = 1.0+ _binning_records.append((0, _z.tolist(), _t, 12))++_temps, _tbo = [1.0, 1.0, 1.0], {}+_bmap = fit_binning_map(_binning_records, _temps, _tbo)+_cut_raw = fit_abstention_thresholds(_binning_records, _temps, _tbo, target_error=0.10)["choice:11+"]+_cut_binned = fit_abstention_thresholds(_binning_records, _temps, _tbo, binning_map=_bmap,+ target_error=0.10)["choice:11+"]+++def _gate_error(cut, binned):+ """Coverage and error among accepted, when the runtime reports binned (or raw) confidences."""+ accepted = wrong = 0+ for qt, z, t, k in _binning_records:+ p = np.exp(np.asarray(z) - max(z))+ p = p / p.sum()+ conf = float(p.max())+ if binned:+ conf = apply_binning_map(conf, "choice:11+", _bmap)+ if conf >= cut:+ accepted += 1+ wrong += int(int(p.argmax()) != int(np.argmax(t)))+ return accepted, (wrong / accepted if accepted else 0.0)+++_acc_raw, _err_raw = _gate_error(_cut_raw, binned=False)+_acc_mismatch, _err_mismatch = _gate_error(_cut_raw, binned=True)+_acc_fit, _err_fit = _gate_error(_cut_binned, binned=True)++check_true("binning/an un-binned cut holds its target on its own scale", _err_raw <= 0.12,+ "cut=%.4f accepted=%d error=%.3f" % (_cut_raw, _acc_raw, _err_raw))+check_true("binning/the same cut stops gating once the runtime bins", _err_mismatch > 0.2,+ "cut=%.4f accepted=%d error=%.3f" % (_cut_raw, _acc_mismatch, _err_mismatch))+check_true("binning/fitting with the map restores the target", _err_fit <= 0.12,+ "cut=%.4f accepted=%d error=%.3f" % (_cut_binned, _acc_fit, _err_fit))+check_true("binning/the two cuts genuinely differ", abs(_cut_binned - _cut_raw) > 1e-9,+ "raw=%.4f binned=%.4f" % (_cut_raw, _cut_binned))+check("binning/no map leaves the fit unchanged",+ fit_abstention_thresholds(_binning_records, _temps, _tbo, binning_map=None,+ target_error=0.10)["choice:11+"], _cut_raw)print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL:print(" FAIL " + f)