Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(calibrate): histogram-binning recalibration for answer_confidence
AlKor133f54a0b2e807
diff --git a/docs/reference/helpers.md b/docs/reference/helpers.mdindex eeda50a0..f638ac5a 100644--- a/docs/reference/helpers.md+++ b/docs/reference/helpers.md@@ -62,6 +62,10 @@::: laya.calibrate.fit_temperature_map+::: laya.calibrate.fit_binning_map++::: laya.calibrate.apply_binning_map+::: laya.common.render_options::: laya.common.proper_rewarddiff --git a/laya/__init__.py b/laya/__init__.pyindex f1ffe8a8..011337c6 100644--- a/laya/__init__.py+++ b/laya/__init__.py@@ -28,6 +28,8 @@ _LAZY_ATTRS = {"fit_temperatures": (".calibrate", "fit_temperatures"),"fit_one_temperature": (".calibrate", "fit_one_temperature"),"fit_temperature_map": (".calibrate", "fit_temperature_map"),+ "fit_binning_map": (".calibrate", "fit_binning_map"),+ "apply_binning_map": (".calibrate", "apply_binning_map"),"proper_reward": (".common", "proper_reward"),"td_lambda_targets": (".common", "td_lambda_targets"),"ece_score": (".common", "ece_score"),@@ -76,6 +78,8 @@ __all__ = ["fit_temperatures","fit_one_temperature","fit_temperature_map",+ "fit_binning_map",+ "apply_binning_map","Router","RouteDecision","DEFAULT_MODELS",diff --git a/laya/calibrate.py b/laya/calibrate.pyindex 3906a5fe..77f8880c 100644--- a/laya/calibrate.py+++ b/laya/calibrate.py@@ -258,6 +258,71 @@ def fit_temperature_map(records: Iterable, compute_ece: bool = False, seed: intfit_temperatures = fit_temperature_map+# Per-bucket floor for histogram-binning recalibration. Higher than the abstention floor because a+# binning map splits each bucket's examples across `bins`, so each bin needs its own sample; lower+# than the temperature floor because binning is a count per bin, not an optimisation.+MIN_BINNING_BUCKET_N = 200+++def fit_binning_map(records: Iterable, temperature: Sequence[float],+ temperature_by_options: Dict[str, float], *, bins: int = 15,+ min_bucket_n: int = MIN_BINNING_BUCKET_N) -> Dict[str, Dict[str, Any]]:+ """Fit a per-`temp_bucket` histogram-binning recalibration map for `answer_confidence`.++ Temperature scaling applies one scalar per bucket; it cannot fix a bucket whose reliability+ curve is not a simple sharpening/softening (the pathological `choice:11+` the shipped English+ checkpoint carries is one). Histogram binning is the non-parametric alternative: split the+ calibrated confidences of a bucket into `bins` equal-width bins over [0, 1], and map every+ confidence that lands in a bin to that bin's empirical accuracy. It needs no monotonicity+ assumption and no extra dependency (NumPy only; isotonic regression would pull in scikit-learn).++ `records` are the same `(qtype, logits, target[, k])` tuples `fit_temperature_map` consumes;+ confidence is the calibrated `max(p)` (logits scaled by the fitted `temperature` /+ `temperature_by_options` first), so a binning map composes on top of a temperature map rather+ than replacing it. Returns `{bucket: {"bins": N, "values": [recalibrated confidence per bin]}}`;+ buckets below `min_bucket_n` are omitted. Apply it with :func:`apply_binning_map`. An empty bin+ (a confidence range the calibration set never produced) maps to its own midpoint, i.e. leaves+ that region unchanged, so an unseen value is never recalibrated to a fabricated 0.+ """+ if bins < 1:+ raise ValueError("bins must be >= 1, got %r" % (bins,))+ recs = _iter_records(records)+ by_bucket: Dict[str, List[Tuple[float, int]]] = {}+ for qt, z, t, k in recs:+ y = int(np.argmax(t[:k]))+ t_scale = temperature_by_options.get(temp_bucket(qt, k), temperature[qt])+ p = _softmax(z[:k], t_scale)+ by_bucket.setdefault(temp_bucket(qt, k), []).append((float(p.max()), int(int(p.argmax()) == y)))+ out: Dict[str, Dict[str, Any]] = {}+ for key, pairs in by_bucket.items():+ if len(pairs) < min_bucket_n:+ continue+ conf = np.asarray([c for c, _ in pairs], dtype=float)+ corr = np.asarray([c for _, c in pairs], dtype=float)+ idx = np.clip((conf * bins).astype(int), 0, bins - 1)+ values = []+ for b in range(bins):+ mask = idx == b+ values.append(float(corr[mask].mean()) if mask.any() else (b + 0.5) / bins)+ out[key] = {"bins": bins, "values": values}+ return out+++def apply_binning_map(confidence: float, bucket: str,+ binning_map: Dict[str, Dict[str, Any]]) -> float:+ """Recalibrate one `answer_confidence` for its option-count `bucket` (`common.temp_bucket`).++ Returns the confidence unchanged when the map has no entry for the bucket, so a bucket the map+ was not fit for passes through rather than being forced to a wrong value.+ """+ entry = binning_map.get(bucket)+ if not entry:+ return float(confidence)+ bins = int(entry["bins"])+ b = min(bins - 1, max(0, int(float(confidence) * bins)))+ return float(entry["values"][b])++def records_from_labeled(agent, pairs: Sequence) -> List[Record]:"""Collect CPU records from `(state, questions, targets)`.diff --git a/tests/test_calibrate.py b/tests/test_calibrate.pyindex e4adb12e..57f8bfd0 100644--- a/tests/test_calibrate.py+++ b/tests/test_calibrate.py@@ -21,17 +21,21 @@ from laya.calibrate import ( # noqa: E402ECE_HOLDOUT_FRAC,MIN_BUCKET_N,MIN_TYPE_N,+ MIN_BINNING_BUCKET_N,_ece_report,_ece_split,_iter_records,+ _softmax,+ apply_binning_map,apply_calibration_payload,+ fit_binning_map,calibration_payload,fit_one_temperature,fit_temperature_map,fit_temperatures,records_from_labeled,)-from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, temp_bucket # noqa: E402+from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, ece_score, temp_bucket # noqa: E402import laya.calibrate as _calibrate # noqa: E402PASS, FAIL = [], []@@ -489,6 +493,43 @@ for name in ("fit_temperatures", "fit_one_temperature", "fit_temperature_map"):)+# --------------------------------------------------------------- histogram-binning recalibration+check("binning/floor", MIN_BINNING_BUCKET_N, 200)+bmap = fit_binning_map(mixed, fitted["temperature"], fitted["temperature_by_options"], bins=10)+check_true("binning/keyed by temp_bucket", {"choice:2", "noul:2", "choice:11+", "choice:3-5"} <= set(bmap), set(bmap))+check_true("binning/omits sub-floor bucket (score:3-5, n=5)", "score:3-5" not in bmap, set(bmap))+_entry = bmap["choice:2"]+check("binning/records bins", _entry["bins"], 10)+check("binning/one value per bin", len(_entry["values"]), 10)+check_true("binning/values are probabilities", all(0.0 <= v <= 1.0 for v in _entry["values"]), _entry["values"])++# apply: a bucket with no map returns the confidence unchanged; edges clamp into range.+check("binning/apply unknown bucket is identity", apply_binning_map(0.73, "choice:6-10", bmap), 0.73)+check_true("binning/apply in range for conf=1.0", 0.0 <= apply_binning_map(1.0, "choice:2", bmap) <= 1.0)+check_true("binning/apply in range for conf=0.0", 0.0 <= apply_binning_map(0.0, "choice:2", bmap) <= 1.0)++# Binning reduces ECE where temperature alone cannot: recalibrate the temperature-scaled+# confidences of a bucket and compare ECE on the same data.+_qt = QTYPES["choice"]+_tscale = fitted["temperature_by_options"].get("choice:2", fitted["temperature"][_qt])+_cal_conf, _corr = [], []+for qt, z, t, k in _iter_records(choice_k2):+ p = _softmax(z[:k], _tscale)+ _cal_conf.append(float(p.max()))+ _corr.append(bool(int(p.argmax()) == int(np.argmax(t[:k]))))+_binned = [apply_binning_map(c, "choice:2", bmap) for c in _cal_conf]+_ece_temp = ece_score(np.asarray(_cal_conf), np.asarray(_corr))+_ece_binned = ece_score(np.asarray(_binned), np.asarray(_corr))+check_true("binning/lowers ECE vs temperature on the bucket",+ _ece_binned < _ece_temp, "temp=%.4f binned=%.4f" % (_ece_temp, _ece_binned))++try:+ fit_binning_map(mixed, fitted["temperature"], fitted["temperature_by_options"], bins=0)+ check_true("binning/bins must be >= 1", False, "no ValueError")+except ValueError:+ check_true("binning/bins must be >= 1", True)++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL:print(" FAIL " + f)