Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(calibrate): histogram-binning recalibration for answer_confidence

AlKor133f54a0b2e807
diff --git a/docs/reference/helpers.md b/docs/reference/helpers.md
index eeda50a0..f638ac5a 100644
--- a/docs/reference/helpers.md
+++ b/docs/reference/helpers.md
@@ -62,6 +62,10 @@
::: laya.calibrate.fit_temperature_map
+::: laya.calibrate.fit_binning_map
+
+::: laya.calibrate.apply_binning_map
+
::: laya.common.render_options
::: laya.common.proper_reward
diff --git a/laya/__init__.py b/laya/__init__.py
index f1ffe8a8..011337c6 100644
--- a/laya/__init__.py
+++ b/laya/__init__.py
@@ -28,6 +28,8 @@ _LAZY_ATTRS = {
"fit_temperatures": (".calibrate", "fit_temperatures"),
"fit_one_temperature": (".calibrate", "fit_one_temperature"),
"fit_temperature_map": (".calibrate", "fit_temperature_map"),
+ "fit_binning_map": (".calibrate", "fit_binning_map"),
+ "apply_binning_map": (".calibrate", "apply_binning_map"),
"proper_reward": (".common", "proper_reward"),
"td_lambda_targets": (".common", "td_lambda_targets"),
"ece_score": (".common", "ece_score"),
@@ -76,6 +78,8 @@ __all__ = [
"fit_temperatures",
"fit_one_temperature",
"fit_temperature_map",
+ "fit_binning_map",
+ "apply_binning_map",
"Router",
"RouteDecision",
"DEFAULT_MODELS",
diff --git a/laya/calibrate.py b/laya/calibrate.py
index 3906a5fe..77f8880c 100644
--- a/laya/calibrate.py
+++ b/laya/calibrate.py
@@ -258,6 +258,71 @@ def fit_temperature_map(records: Iterable, compute_ece: bool = False, seed: int
fit_temperatures = fit_temperature_map
+# Per-bucket floor for histogram-binning recalibration. Higher than the abstention floor because a
+# binning map splits each bucket's examples across `bins`, so each bin needs its own sample; lower
+# than the temperature floor because binning is a count per bin, not an optimisation.
+MIN_BINNING_BUCKET_N = 200
+
+
+def fit_binning_map(records: Iterable, temperature: Sequence[float],
+ temperature_by_options: Dict[str, float], *, bins: int = 15,
+ min_bucket_n: int = MIN_BINNING_BUCKET_N) -> Dict[str, Dict[str, Any]]:
+ """Fit a per-`temp_bucket` histogram-binning recalibration map for `answer_confidence`.
+
+ Temperature scaling applies one scalar per bucket; it cannot fix a bucket whose reliability
+ curve is not a simple sharpening/softening (the pathological `choice:11+` the shipped English
+ checkpoint carries is one). Histogram binning is the non-parametric alternative: split the
+ calibrated confidences of a bucket into `bins` equal-width bins over [0, 1], and map every
+ confidence that lands in a bin to that bin's empirical accuracy. It needs no monotonicity
+ assumption and no extra dependency (NumPy only; isotonic regression would pull in scikit-learn).
+
+ `records` are the same `(qtype, logits, target[, k])` tuples `fit_temperature_map` consumes;
+ confidence is the calibrated `max(p)` (logits scaled by the fitted `temperature` /
+ `temperature_by_options` first), so a binning map composes on top of a temperature map rather
+ than replacing it. Returns `{bucket: {"bins": N, "values": [recalibrated confidence per bin]}}`;
+ buckets below `min_bucket_n` are omitted. Apply it with :func:`apply_binning_map`. An empty bin
+ (a confidence range the calibration set never produced) maps to its own midpoint, i.e. leaves
+ that region unchanged, so an unseen value is never recalibrated to a fabricated 0.
+ """
+ if bins < 1:
+ raise ValueError("bins must be >= 1, got %r" % (bins,))
+ recs = _iter_records(records)
+ by_bucket: Dict[str, List[Tuple[float, int]]] = {}
+ for qt, z, t, k in recs:
+ y = int(np.argmax(t[:k]))
+ t_scale = temperature_by_options.get(temp_bucket(qt, k), temperature[qt])
+ p = _softmax(z[:k], t_scale)
+ by_bucket.setdefault(temp_bucket(qt, k), []).append((float(p.max()), int(int(p.argmax()) == y)))
+ out: Dict[str, Dict[str, Any]] = {}
+ for key, pairs in by_bucket.items():
+ if len(pairs) < min_bucket_n:
+ continue
+ conf = np.asarray([c for c, _ in pairs], dtype=float)
+ corr = np.asarray([c for _, c in pairs], dtype=float)
+ idx = np.clip((conf * bins).astype(int), 0, bins - 1)
+ values = []
+ for b in range(bins):
+ mask = idx == b
+ values.append(float(corr[mask].mean()) if mask.any() else (b + 0.5) / bins)
+ out[key] = {"bins": bins, "values": values}
+ return out
+
+
+def apply_binning_map(confidence: float, bucket: str,
+ binning_map: Dict[str, Dict[str, Any]]) -> float:
+ """Recalibrate one `answer_confidence` for its option-count `bucket` (`common.temp_bucket`).
+
+ Returns the confidence unchanged when the map has no entry for the bucket, so a bucket the map
+ was not fit for passes through rather than being forced to a wrong value.
+ """
+ entry = binning_map.get(bucket)
+ if not entry:
+ return float(confidence)
+ bins = int(entry["bins"])
+ b = min(bins - 1, max(0, int(float(confidence) * bins)))
+ return float(entry["values"][b])
+
+
def records_from_labeled(agent, pairs: Sequence) -> List[Record]:
"""Collect CPU records from `(state, questions, targets)`.
diff --git a/tests/test_calibrate.py b/tests/test_calibrate.py
index e4adb12e..57f8bfd0 100644
--- a/tests/test_calibrate.py
+++ b/tests/test_calibrate.py
@@ -21,17 +21,21 @@ from laya.calibrate import ( # noqa: E402
ECE_HOLDOUT_FRAC,
MIN_BUCKET_N,
MIN_TYPE_N,
+ MIN_BINNING_BUCKET_N,
_ece_report,
_ece_split,
_iter_records,
+ _softmax,
+ apply_binning_map,
apply_calibration_payload,
+ fit_binning_map,
calibration_payload,
fit_one_temperature,
fit_temperature_map,
fit_temperatures,
records_from_labeled,
)
-from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, temp_bucket # noqa: E402
+from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, ece_score, temp_bucket # noqa: E402
import laya.calibrate as _calibrate # noqa: E402
PASS, FAIL = [], []
@@ -489,6 +493,43 @@ for name in ("fit_temperatures", "fit_one_temperature", "fit_temperature_map"):
)
+# --------------------------------------------------------------- histogram-binning recalibration
+check("binning/floor", MIN_BINNING_BUCKET_N, 200)
+bmap = fit_binning_map(mixed, fitted["temperature"], fitted["temperature_by_options"], bins=10)
+check_true("binning/keyed by temp_bucket", {"choice:2", "noul:2", "choice:11+", "choice:3-5"} <= set(bmap), set(bmap))
+check_true("binning/omits sub-floor bucket (score:3-5, n=5)", "score:3-5" not in bmap, set(bmap))
+_entry = bmap["choice:2"]
+check("binning/records bins", _entry["bins"], 10)
+check("binning/one value per bin", len(_entry["values"]), 10)
+check_true("binning/values are probabilities", all(0.0 <= v <= 1.0 for v in _entry["values"]), _entry["values"])
+
+# apply: a bucket with no map returns the confidence unchanged; edges clamp into range.
+check("binning/apply unknown bucket is identity", apply_binning_map(0.73, "choice:6-10", bmap), 0.73)
+check_true("binning/apply in range for conf=1.0", 0.0 <= apply_binning_map(1.0, "choice:2", bmap) <= 1.0)
+check_true("binning/apply in range for conf=0.0", 0.0 <= apply_binning_map(0.0, "choice:2", bmap) <= 1.0)
+
+# Binning reduces ECE where temperature alone cannot: recalibrate the temperature-scaled
+# confidences of a bucket and compare ECE on the same data.
+_qt = QTYPES["choice"]
+_tscale = fitted["temperature_by_options"].get("choice:2", fitted["temperature"][_qt])
+_cal_conf, _corr = [], []
+for qt, z, t, k in _iter_records(choice_k2):
+ p = _softmax(z[:k], _tscale)
+ _cal_conf.append(float(p.max()))
+ _corr.append(bool(int(p.argmax()) == int(np.argmax(t[:k]))))
+_binned = [apply_binning_map(c, "choice:2", bmap) for c in _cal_conf]
+_ece_temp = ece_score(np.asarray(_cal_conf), np.asarray(_corr))
+_ece_binned = ece_score(np.asarray(_binned), np.asarray(_corr))
+check_true("binning/lowers ECE vs temperature on the bucket",
+ _ece_binned < _ece_temp, "temp=%.4f binned=%.4f" % (_ece_temp, _ece_binned))
+
+try:
+ fit_binning_map(mixed, fitted["temperature"], fitted["temperature_by_options"], bins=0)
+ check_true("binning/bins must be >= 1", False, "no ValueError")
+except ValueError:
+ check_true("binning/bins must be >= 1", True)
+
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL " + f)