Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #871 from AlKor13/feat-histogram-binning

Nandakishor98ebcbb75518
diff --cc docs/reference/helpers.md
index bdedf8be,f638ac5a..697ba4be
--- a/docs/reference/helpers.md
+++ b/docs/reference/helpers.md
@@@ -66,7 -62,9 +66,10 @@@
::: laya.calibrate.fit_temperature_map
+::: laya.calibrate.fit_abstention_thresholds
+ ::: laya.calibrate.fit_binning_map
+
+ ::: laya.calibrate.apply_binning_map
::: laya.common.render_options
diff --cc laya/__init__.py
index 702371c4,011337c6..c18c8234
--- a/laya/__init__.py
+++ b/laya/__init__.py
@@@ -28,7 -28,8 +28,9 @@@ _LAZY_ATTRS =
"fit_temperatures": (".calibrate", "fit_temperatures"),
"fit_one_temperature": (".calibrate", "fit_one_temperature"),
"fit_temperature_map": (".calibrate", "fit_temperature_map"),
+ "fit_abstention_thresholds": (".calibrate", "fit_abstention_thresholds"),
+ "fit_binning_map": (".calibrate", "fit_binning_map"),
+ "apply_binning_map": (".calibrate", "apply_binning_map"),
"proper_reward": (".common", "proper_reward"),
"td_lambda_targets": (".common", "td_lambda_targets"),
"ece_score": (".common", "ece_score"),
@@@ -79,7 -78,8 +81,9 @@@ __all__ =
"fit_temperatures",
"fit_one_temperature",
"fit_temperature_map",
+ "fit_abstention_thresholds",
+ "fit_binning_map",
+ "apply_binning_map",
"Router",
"RouteDecision",
"DEFAULT_MODELS",
diff --cc laya/calibrate.py
index e45aecfa,77f8880c..443d642a
--- a/laya/calibrate.py
+++ b/laya/calibrate.py
@@@ -297,85 -258,71 +297,150 @@@ def fit_temperature_map(records: Iterab
fit_temperatures = fit_temperature_map
+# Per-bucket floor for abstention thresholds. Lower than `MIN_BUCKET_N` (which governs temperature
+# fitting): a threshold is a single order statistic of the calibrated confidences, so it stabilises
+# on far fewer examples than an LBFGS temperature does. Buckets below this are omitted; the caller's
+# scalar `min_confidence` (or the map's "default") covers them.
+MIN_ABSTAIN_BUCKET_N = 100
+
+
+def _select_abstention_threshold(pairs: Sequence[Tuple[float, int]], target_error: float,
+ conservative: bool) -> float:
+ """Smallest confidence `tau` whose accepted set (`conf >= tau`) keeps error <= `target_error`.
+
+ `pairs` are `(confidence, correct)`. Sweeping from the most confident down maximises coverage
+ at the target risk (the selective-classification / split-conformal cut). `conservative` adds one
+ pseudo-error so a bucket does not clear the gate on a lucky short run. Returns 1.0 when no cut
+ holds the risk -- the bucket is too unreliable to accept anything short of a reported certainty.
+ """
+ ordered = sorted(pairs, key=lambda cc: cc[0], reverse=True)
+ levels = sorted({c for c, _ in ordered}, reverse=True)
+ n = len(ordered)
+ n_acc = n_err = idx = 0
+ best = None
+ # Evaluate the error over the WHOLE accepted set {conf >= tau} at each distinct level, not
+ # incrementally within a tie: a threshold accepts every answer at its own confidence, so a tie
+ # group's errors must all be counted before the level is judged (else the cut sinks into a bad
+ # cohort on its first few correct members).
+ for tau in levels:
+ while idx < n and ordered[idx][0] >= tau:
+ n_acc += 1
+ n_err += 0 if ordered[idx][1] else 1
+ idx += 1
+ rate = (n_err + 1.0) / (n_acc + 1.0) if conservative else (n_err / n_acc)
+ if n_acc > 0 and rate <= target_error:
+ best = tau
+ if best is None:
+ return 1.0
+ return float(min(1.0, max(0.0, best)))
+
+
+def fit_abstention_thresholds(records: Iterable, temperature: Sequence[float],
+ temperature_by_options: Dict[str, float], *,
+ target_error: float = 0.10,
+ min_bucket_n: int = MIN_ABSTAIN_BUCKET_N,
+ conservative: bool = True) -> Dict[str, float]:
+ """Fit a per-`temp_bucket` abstention threshold so a gate keeps a target error in every bucket.
+
+ A single `min_confidence` does not transfer across option counts (#394): the calibrated
+ confidence of a 2-option and a 12-option answer live on different scales, so one cut over- or
+ under-abstains depending on the question. This fits one cut per bucket instead, keyed exactly
+ like `temperature_by_options` (`common.temp_bucket`, e.g. ``"choice:3-5"``), and the result is a
+ `min_confidence` map that :func:`laya.confidence.check_min_confidence` /
+ :func:`laya.confidence.apply_confidence_gate` accept directly.
+
+ `records` are the same `(qtype, logits, target[, k])` tuples `fit_temperature_map` consumes
+ (`records_from_labeled` builds them). Confidence is the **calibrated** `max(p)` -- the logits are
+ scaled by the fitted `temperature` / `temperature_by_options` first, so thresholds and the
+ numbers the runtime reports are on the same scale. `target_error` is the tolerated error among
+ accepted answers; `min_bucket_n` omits buckets too small to fit, and `conservative` adds a
+ one-sample margin. The thresholds are empirical cuts on the calibration set, not a formal
+ coverage guarantee -- validate on held-out data (`fit_temperature_map(..., compute_ece=True)`
+ gives a held-out split) for a production gate.
+ """
+ if not 0.0 <= target_error <= 1.0:
+ raise ValueError("target_error must be in [0.0, 1.0], got %r" % (target_error,))
+ recs = _iter_records(records)
+ by_bucket: Dict[str, List[Tuple[float, int]]] = {}
+ for qt, z, t, k in recs:
+ y = int(np.argmax(t[:k]))
+ t_scale = temperature_by_options.get(temp_bucket(qt, k), temperature[qt])
+ p = _softmax(z[:k], t_scale)
+ by_bucket.setdefault(temp_bucket(qt, k), []).append((float(p.max()), int(int(p.argmax()) == y)))
+ out: Dict[str, float] = {}
+ for key, pairs in by_bucket.items():
+ if len(pairs) < min_bucket_n:
+ continue
+ out[key] = _select_abstention_threshold(pairs, target_error, conservative)
+ return out
+
+
+ # Per-bucket floor for histogram-binning recalibration. Higher than the abstention floor because a
+ # binning map splits each bucket's examples across `bins`, so each bin needs its own sample; lower
+ # than the temperature floor because binning is a count per bin, not an optimisation.
+ MIN_BINNING_BUCKET_N = 200
+
+
+ def fit_binning_map(records: Iterable, temperature: Sequence[float],
+ temperature_by_options: Dict[str, float], *, bins: int = 15,
+ min_bucket_n: int = MIN_BINNING_BUCKET_N) -> Dict[str, Dict[str, Any]]:
+ """Fit a per-`temp_bucket` histogram-binning recalibration map for `answer_confidence`.
+
+ Temperature scaling applies one scalar per bucket; it cannot fix a bucket whose reliability
+ curve is not a simple sharpening/softening (the pathological `choice:11+` the shipped English
+ checkpoint carries is one). Histogram binning is the non-parametric alternative: split the
+ calibrated confidences of a bucket into `bins` equal-width bins over [0, 1], and map every
+ confidence that lands in a bin to that bin's empirical accuracy. It needs no monotonicity
+ assumption and no extra dependency (NumPy only; isotonic regression would pull in scikit-learn).
+
+ `records` are the same `(qtype, logits, target[, k])` tuples `fit_temperature_map` consumes;
+ confidence is the calibrated `max(p)` (logits scaled by the fitted `temperature` /
+ `temperature_by_options` first), so a binning map composes on top of a temperature map rather
+ than replacing it. Returns `{bucket: {"bins": N, "values": [recalibrated confidence per bin]}}`;
+ buckets below `min_bucket_n` are omitted. Apply it with :func:`apply_binning_map`. An empty bin
+ (a confidence range the calibration set never produced) maps to its own midpoint, i.e. leaves
+ that region unchanged, so an unseen value is never recalibrated to a fabricated 0.
+ """
+ if bins < 1:
+ raise ValueError("bins must be >= 1, got %r" % (bins,))
+ recs = _iter_records(records)
+ by_bucket: Dict[str, List[Tuple[float, int]]] = {}
+ for qt, z, t, k in recs:
+ y = int(np.argmax(t[:k]))
+ t_scale = temperature_by_options.get(temp_bucket(qt, k), temperature[qt])
+ p = _softmax(z[:k], t_scale)
+ by_bucket.setdefault(temp_bucket(qt, k), []).append((float(p.max()), int(int(p.argmax()) == y)))
+ out: Dict[str, Dict[str, Any]] = {}
+ for key, pairs in by_bucket.items():
+ if len(pairs) < min_bucket_n:
+ continue
+ conf = np.asarray([c for c, _ in pairs], dtype=float)
+ corr = np.asarray([c for _, c in pairs], dtype=float)
+ idx = np.clip((conf * bins).astype(int), 0, bins - 1)
+ values = []
+ for b in range(bins):
+ mask = idx == b
+ values.append(float(corr[mask].mean()) if mask.any() else (b + 0.5) / bins)
+ out[key] = {"bins": bins, "values": values}
+ return out
+
+
+ def apply_binning_map(confidence: float, bucket: str,
+ binning_map: Dict[str, Dict[str, Any]]) -> float:
+ """Recalibrate one `answer_confidence` for its option-count `bucket` (`common.temp_bucket`).
+
+ Returns the confidence unchanged when the map has no entry for the bucket, so a bucket the map
+ was not fit for passes through rather than being forced to a wrong value.
+ """
+ entry = binning_map.get(bucket)
+ if not entry:
+ return float(confidence)
+ bins = int(entry["bins"])
+ b = min(bins - 1, max(0, int(float(confidence) * bins)))
+ return float(entry["values"][b])
+
+
def records_from_labeled(agent, pairs: Sequence) -> List[Record]:
"""Collect CPU records from `(state, questions, targets)`.
diff --cc tests/test_calibrate.py
index 8c69b976,57f8bfd0..62769ee0
--- a/tests/test_calibrate.py
+++ b/tests/test_calibrate.py
@@@ -35,9 -35,8 +39,9 @@@ from laya.calibrate import ( # noqa: E
fit_temperatures,
records_from_labeled,
)
- from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, temp_bucket # noqa: E402
+ from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, ece_score, temp_bucket # noqa: E402
import laya.calibrate as _calibrate # noqa: E402
+import laya.agent as _agent_module # noqa: E402
PASS, FAIL = [], []