Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(common): refuse a bins count ECE cannot histogram
Aashish6b37ae711a82
diff --git a/laya/common.py b/laya/common.pyindex 15fbc3a5..8ec6a181 100644--- a/laya/common.py+++ b/laya/common.py@@ -7,6 +7,7 @@ import warningsfrom contextlib import nullcontextfrom contextvars import ContextVarfrom functools import wraps+from numbers import Integralfrom typing import Any, Dict, List, Optional, Sequence, Unionimport numpy as np@@ -764,7 +765,18 @@ def td_lambda_targets(p_true: torch.Tensor, batch: Dict, lam: float = 1.0) -> todef ece_score(conf: np.ndarray, correct: np.ndarray, bins: int = 15) -> float:- """Expected Calibration Error across confidence bins."""+ """Expected Calibration Error across confidence bins.++ `bins` is the number of histogram buckets and must be an integer of at least 1. It is refused+ rather than misread because `np.linspace(0, 1, bins + 1)` yields no interior edge for `0` and+ `-1`: the loop body never runs and the function returns `0.0`, the best score ECE can report,+ for a distribution that is anything but well calibrated. A count that arrives from a config+ file as `0` therefore turns a calibration gate into one that always passes. `True` is refused+ with the same argument -- it would silently histogram into a single bucket. Any other+ `Integral` is accepted, a `numpy.int64` among them, since the arrays here are numpy's own.+ """+ if isinstance(bins, bool) or not isinstance(bins, Integral) or bins < 1:+ raise ValueError("bins must be an integer of at least 1, got %r" % (bins,))if len(conf) == 0:return float("nan")edges = np.linspace(0, 1, bins + 1)diff --git a/tests/test_training.py b/tests/test_training.pyindex 4075e87f..d1555bbb 100644--- a/tests/test_training.py+++ b/tests/test_training.py@@ -203,6 +203,57 @@ check_true("ece/overconfidence raises ECE", ece_score(conf_b, base_correct) > ec# bins argument changes granularity, not correctness of the perfect caseclose("ece/bins=5 perfectly calibrated", ece_score(conf, correct, bins=5), 0.0, tol=1e-9)+# --------------------------------------------------------------- ece_score's bins knob+# The histogram is sized by `bins`, and a count that cannot size one used to be misread rather+# than refused: `np.linspace(0, 1, 0 + 1)` has no interior edge, so the loop body never ran and+# the function returned 0.0 -- the best score ECE can report -- for a distribution that is badly+# calibrated. A caller that gates a build on ECE therefore passes unconditionally on a config+# value of 0, which is exactly the number a missing or zeroed field tends to be.+_mis = (np.full(10, 0.99), np.array([1] * 5 + [0] * 5))+close("ece/overconfident and half right reports the gap", ece_score(*_mis), 0.49, tol=1e-9)+_ECE_REFUSAL = "bins must be an integer of at least 1"+++def _ece_refuses_bins(value):+ try:+ got = ece_score(_mis[0], _mis[1], bins=value)+ except ValueError as exc:+ check_true("ece/refuses bins=%r" % (value,),+ _ECE_REFUSAL in str(exc) and "got %r" % (value,) in str(exc), str(exc))+ return+ except BaseException as exc: # noqa: BLE001 -- the bare TypeError these used to leak+ FAIL.append("ece/refuses bins=%r: raised %r, want ValueError" % (value, exc))+ return+ FAIL.append("ece/refuses bins=%r: returned %r" % (value, got))+++# 0 and -1 leave no interior edge so the sum stays 0.0, -5 reaches np.linspace and gets numpy's+# complaint about a negative sample count, and the rest used to raise a bare TypeError.+for _b in (0, -1, -5, True, False, 1.5, "15", None, np.float64(3.0)):+ _ece_refuses_bins(_b)++# the count is checked before the sample, so an empty run cannot smuggle a bad knob past the+# branch that returns nan+try:+ ece_score(np.array([]), np.array([]), bins=0)+ FAIL.append("ece/refuses bins=0 with no samples: returned instead of raising")+except ValueError as exc:+ check_true("ece/refuses bins=0 with no samples", _ECE_REFUSAL in str(exc), str(exc))++# numpy integer counts are the ones that arrive with the arrays this metric is handed+check("ece/bins=5 as np.int64 matches int",+ ece_score(*_mis, bins=np.int64(5)), ece_score(*_mis, bins=5))+close("ece/bins=1 is the mean gap over the whole range", ece_score(*_mis, bins=1), 0.49, tol=1e-9)++# laya.evals.ece delegates here, so the CLI's gate inherits the refusal+from laya.evals import ece as _evals_ece # noqa: E402++try:+ _evals_ece([0.99] * 10, [True] * 5 + [False] * 5, bins=0)+ FAIL.append("ece/laya.evals.ece inherits the refusal: returned instead of raising")+except ValueError as exc:+ check_true("ece/laya.evals.ece inherits the refusal", _ECE_REFUSAL in str(exc), str(exc))+# =============================================================== confidence_from_probsclose("confidence/uniform over 2 is 0", confidence_from_probs(np.array([0.5, 0.5]), 2), 0.0)