Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
Merge pull request #871 from AlKor13/feat-histogram-binning
Nandakishor98ebcbb75518
diff --cc docs/reference/helpers.mdindex bdedf8be,f638ac5a..697ba4be--- a/docs/reference/helpers.md+++ b/docs/reference/helpers.md@@@ -66,7 -62,9 +66,10 @@@::: laya.calibrate.fit_temperature_map+::: laya.calibrate.fit_abstention_thresholds+ ::: laya.calibrate.fit_binning_map++ ::: laya.calibrate.apply_binning_map::: laya.common.render_optionsdiff --cc laya/__init__.pyindex 702371c4,011337c6..c18c8234--- a/laya/__init__.py+++ b/laya/__init__.py@@@ -28,7 -28,8 +28,9 @@@ _LAZY_ATTRS ="fit_temperatures": (".calibrate", "fit_temperatures"),"fit_one_temperature": (".calibrate", "fit_one_temperature"),"fit_temperature_map": (".calibrate", "fit_temperature_map"),+ "fit_abstention_thresholds": (".calibrate", "fit_abstention_thresholds"),+ "fit_binning_map": (".calibrate", "fit_binning_map"),+ "apply_binning_map": (".calibrate", "apply_binning_map"),"proper_reward": (".common", "proper_reward"),"td_lambda_targets": (".common", "td_lambda_targets"),"ece_score": (".common", "ece_score"),@@@ -79,7 -78,8 +81,9 @@@ __all__ ="fit_temperatures","fit_one_temperature","fit_temperature_map",+ "fit_abstention_thresholds",+ "fit_binning_map",+ "apply_binning_map","Router","RouteDecision","DEFAULT_MODELS",diff --cc laya/calibrate.pyindex e45aecfa,77f8880c..443d642a--- a/laya/calibrate.py+++ b/laya/calibrate.py@@@ -297,85 -258,71 +297,150 @@@ def fit_temperature_map(records: Iterabfit_temperatures = fit_temperature_map+# Per-bucket floor for abstention thresholds. Lower than `MIN_BUCKET_N` (which governs temperature+# fitting): a threshold is a single order statistic of the calibrated confidences, so it stabilises+# on far fewer examples than an LBFGS temperature does. Buckets below this are omitted; the caller's+# scalar `min_confidence` (or the map's "default") covers them.+MIN_ABSTAIN_BUCKET_N = 100+++def _select_abstention_threshold(pairs: Sequence[Tuple[float, int]], target_error: float,+ conservative: bool) -> float:+ """Smallest confidence `tau` whose accepted set (`conf >= tau`) keeps error <= `target_error`.++ `pairs` are `(confidence, correct)`. Sweeping from the most confident down maximises coverage+ at the target risk (the selective-classification / split-conformal cut). `conservative` adds one+ pseudo-error so a bucket does not clear the gate on a lucky short run. Returns 1.0 when no cut+ holds the risk -- the bucket is too unreliable to accept anything short of a reported certainty.+ """+ ordered = sorted(pairs, key=lambda cc: cc[0], reverse=True)+ levels = sorted({c for c, _ in ordered}, reverse=True)+ n = len(ordered)+ n_acc = n_err = idx = 0+ best = None+ # Evaluate the error over the WHOLE accepted set {conf >= tau} at each distinct level, not+ # incrementally within a tie: a threshold accepts every answer at its own confidence, so a tie+ # group's errors must all be counted before the level is judged (else the cut sinks into a bad+ # cohort on its first few correct members).+ for tau in levels:+ while idx < n and ordered[idx][0] >= tau:+ n_acc += 1+ n_err += 0 if ordered[idx][1] else 1+ idx += 1+ rate = (n_err + 1.0) / (n_acc + 1.0) if conservative else (n_err / n_acc)+ if n_acc > 0 and rate <= target_error:+ best = tau+ if best is None:+ return 1.0+ return float(min(1.0, max(0.0, best)))+++def fit_abstention_thresholds(records: Iterable, temperature: Sequence[float],+ temperature_by_options: Dict[str, float], *,+ target_error: float = 0.10,+ min_bucket_n: int = MIN_ABSTAIN_BUCKET_N,+ conservative: bool = True) -> Dict[str, float]:+ """Fit a per-`temp_bucket` abstention threshold so a gate keeps a target error in every bucket.++ A single `min_confidence` does not transfer across option counts (#394): the calibrated+ confidence of a 2-option and a 12-option answer live on different scales, so one cut over- or+ under-abstains depending on the question. This fits one cut per bucket instead, keyed exactly+ like `temperature_by_options` (`common.temp_bucket`, e.g. ``"choice:3-5"``), and the result is a+ `min_confidence` map that :func:`laya.confidence.check_min_confidence` /+ :func:`laya.confidence.apply_confidence_gate` accept directly.++ `records` are the same `(qtype, logits, target[, k])` tuples `fit_temperature_map` consumes+ (`records_from_labeled` builds them). Confidence is the **calibrated** `max(p)` -- the logits are+ scaled by the fitted `temperature` / `temperature_by_options` first, so thresholds and the+ numbers the runtime reports are on the same scale. `target_error` is the tolerated error among+ accepted answers; `min_bucket_n` omits buckets too small to fit, and `conservative` adds a+ one-sample margin. The thresholds are empirical cuts on the calibration set, not a formal+ coverage guarantee -- validate on held-out data (`fit_temperature_map(..., compute_ece=True)`+ gives a held-out split) for a production gate.+ """+ if not 0.0 <= target_error <= 1.0:+ raise ValueError("target_error must be in [0.0, 1.0], got %r" % (target_error,))+ recs = _iter_records(records)+ by_bucket: Dict[str, List[Tuple[float, int]]] = {}+ for qt, z, t, k in recs:+ y = int(np.argmax(t[:k]))+ t_scale = temperature_by_options.get(temp_bucket(qt, k), temperature[qt])+ p = _softmax(z[:k], t_scale)+ by_bucket.setdefault(temp_bucket(qt, k), []).append((float(p.max()), int(int(p.argmax()) == y)))+ out: Dict[str, float] = {}+ for key, pairs in by_bucket.items():+ if len(pairs) < min_bucket_n:+ continue+ out[key] = _select_abstention_threshold(pairs, target_error, conservative)+ return out+++ # Per-bucket floor for histogram-binning recalibration. Higher than the abstention floor because a+ # binning map splits each bucket's examples across `bins`, so each bin needs its own sample; lower+ # than the temperature floor because binning is a count per bin, not an optimisation.+ MIN_BINNING_BUCKET_N = 200+++ def fit_binning_map(records: Iterable, temperature: Sequence[float],+ temperature_by_options: Dict[str, float], *, bins: int = 15,+ min_bucket_n: int = MIN_BINNING_BUCKET_N) -> Dict[str, Dict[str, Any]]:+ """Fit a per-`temp_bucket` histogram-binning recalibration map for `answer_confidence`.++ Temperature scaling applies one scalar per bucket; it cannot fix a bucket whose reliability+ curve is not a simple sharpening/softening (the pathological `choice:11+` the shipped English+ checkpoint carries is one). Histogram binning is the non-parametric alternative: split the+ calibrated confidences of a bucket into `bins` equal-width bins over [0, 1], and map every+ confidence that lands in a bin to that bin's empirical accuracy. It needs no monotonicity+ assumption and no extra dependency (NumPy only; isotonic regression would pull in scikit-learn).++ `records` are the same `(qtype, logits, target[, k])` tuples `fit_temperature_map` consumes;+ confidence is the calibrated `max(p)` (logits scaled by the fitted `temperature` /+ `temperature_by_options` first), so a binning map composes on top of a temperature map rather+ than replacing it. Returns `{bucket: {"bins": N, "values": [recalibrated confidence per bin]}}`;+ buckets below `min_bucket_n` are omitted. Apply it with :func:`apply_binning_map`. An empty bin+ (a confidence range the calibration set never produced) maps to its own midpoint, i.e. leaves+ that region unchanged, so an unseen value is never recalibrated to a fabricated 0.+ """+ if bins < 1:+ raise ValueError("bins must be >= 1, got %r" % (bins,))+ recs = _iter_records(records)+ by_bucket: Dict[str, List[Tuple[float, int]]] = {}+ for qt, z, t, k in recs:+ y = int(np.argmax(t[:k]))+ t_scale = temperature_by_options.get(temp_bucket(qt, k), temperature[qt])+ p = _softmax(z[:k], t_scale)+ by_bucket.setdefault(temp_bucket(qt, k), []).append((float(p.max()), int(int(p.argmax()) == y)))+ out: Dict[str, Dict[str, Any]] = {}+ for key, pairs in by_bucket.items():+ if len(pairs) < min_bucket_n:+ continue+ conf = np.asarray([c for c, _ in pairs], dtype=float)+ corr = np.asarray([c for _, c in pairs], dtype=float)+ idx = np.clip((conf * bins).astype(int), 0, bins - 1)+ values = []+ for b in range(bins):+ mask = idx == b+ values.append(float(corr[mask].mean()) if mask.any() else (b + 0.5) / bins)+ out[key] = {"bins": bins, "values": values}+ return out+++ def apply_binning_map(confidence: float, bucket: str,+ binning_map: Dict[str, Dict[str, Any]]) -> float:+ """Recalibrate one `answer_confidence` for its option-count `bucket` (`common.temp_bucket`).++ Returns the confidence unchanged when the map has no entry for the bucket, so a bucket the map+ was not fit for passes through rather than being forced to a wrong value.+ """+ entry = binning_map.get(bucket)+ if not entry:+ return float(confidence)+ bins = int(entry["bins"])+ b = min(bins - 1, max(0, int(float(confidence) * bins)))+ return float(entry["values"][b])++[email protected]_grad()def records_from_labeled(agent, pairs: Sequence) -> List[Record]:"""Collect CPU records from `(state, questions, targets)`.diff --cc tests/test_calibrate.pyindex 8c69b976,57f8bfd0..62769ee0--- a/tests/test_calibrate.py+++ b/tests/test_calibrate.py@@@ -35,9 -35,8 +39,9 @@@ from laya.calibrate import ( # noqa: Efit_temperatures,records_from_labeled,)- from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, temp_bucket # noqa: E402+ from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, ece_score, temp_bucket # noqa: E402import laya.calibrate as _calibrate # noqa: E402+import laya.agent as _agent_module # noqa: E402PASS, FAIL = [], []