Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(common): refuse a bins count ECE cannot histogram

Aashish6b37ae711a82
diff --git a/laya/common.py b/laya/common.py
index 15fbc3a5..8ec6a181 100644
--- a/laya/common.py
+++ b/laya/common.py
@@ -7,6 +7,7 @@ import warnings
from contextlib import nullcontext
from contextvars import ContextVar
from functools import wraps
+from numbers import Integral
from typing import Any, Dict, List, Optional, Sequence, Union
import numpy as np
@@ -764,7 +765,18 @@ def td_lambda_targets(p_true: torch.Tensor, batch: Dict, lam: float = 1.0) -> to
def ece_score(conf: np.ndarray, correct: np.ndarray, bins: int = 15) -> float:
- """Expected Calibration Error across confidence bins."""
+ """Expected Calibration Error across confidence bins.
+
+ `bins` is the number of histogram buckets and must be an integer of at least 1. It is refused
+ rather than misread because `np.linspace(0, 1, bins + 1)` yields no interior edge for `0` and
+ `-1`: the loop body never runs and the function returns `0.0`, the best score ECE can report,
+ for a distribution that is anything but well calibrated. A count that arrives from a config
+ file as `0` therefore turns a calibration gate into one that always passes. `True` is refused
+ with the same argument -- it would silently histogram into a single bucket. Any other
+ `Integral` is accepted, a `numpy.int64` among them, since the arrays here are numpy's own.
+ """
+ if isinstance(bins, bool) or not isinstance(bins, Integral) or bins < 1:
+ raise ValueError("bins must be an integer of at least 1, got %r" % (bins,))
if len(conf) == 0:
return float("nan")
edges = np.linspace(0, 1, bins + 1)
diff --git a/tests/test_training.py b/tests/test_training.py
index 4075e87f..d1555bbb 100644
--- a/tests/test_training.py
+++ b/tests/test_training.py
@@ -203,6 +203,57 @@ check_true("ece/overconfidence raises ECE", ece_score(conf_b, base_correct) > ec
# bins argument changes granularity, not correctness of the perfect case
close("ece/bins=5 perfectly calibrated", ece_score(conf, correct, bins=5), 0.0, tol=1e-9)
+# --------------------------------------------------------------- ece_score's bins knob
+# The histogram is sized by `bins`, and a count that cannot size one used to be misread rather
+# than refused: `np.linspace(0, 1, 0 + 1)` has no interior edge, so the loop body never ran and
+# the function returned 0.0 -- the best score ECE can report -- for a distribution that is badly
+# calibrated. A caller that gates a build on ECE therefore passes unconditionally on a config
+# value of 0, which is exactly the number a missing or zeroed field tends to be.
+_mis = (np.full(10, 0.99), np.array([1] * 5 + [0] * 5))
+close("ece/overconfident and half right reports the gap", ece_score(*_mis), 0.49, tol=1e-9)
+_ECE_REFUSAL = "bins must be an integer of at least 1"
+
+
+def _ece_refuses_bins(value):
+ try:
+ got = ece_score(_mis[0], _mis[1], bins=value)
+ except ValueError as exc:
+ check_true("ece/refuses bins=%r" % (value,),
+ _ECE_REFUSAL in str(exc) and "got %r" % (value,) in str(exc), str(exc))
+ return
+ except BaseException as exc: # noqa: BLE001 -- the bare TypeError these used to leak
+ FAIL.append("ece/refuses bins=%r: raised %r, want ValueError" % (value, exc))
+ return
+ FAIL.append("ece/refuses bins=%r: returned %r" % (value, got))
+
+
+# 0 and -1 leave no interior edge so the sum stays 0.0, -5 reaches np.linspace and gets numpy's
+# complaint about a negative sample count, and the rest used to raise a bare TypeError.
+for _b in (0, -1, -5, True, False, 1.5, "15", None, np.float64(3.0)):
+ _ece_refuses_bins(_b)
+
+# the count is checked before the sample, so an empty run cannot smuggle a bad knob past the
+# branch that returns nan
+try:
+ ece_score(np.array([]), np.array([]), bins=0)
+ FAIL.append("ece/refuses bins=0 with no samples: returned instead of raising")
+except ValueError as exc:
+ check_true("ece/refuses bins=0 with no samples", _ECE_REFUSAL in str(exc), str(exc))
+
+# numpy integer counts are the ones that arrive with the arrays this metric is handed
+check("ece/bins=5 as np.int64 matches int",
+ ece_score(*_mis, bins=np.int64(5)), ece_score(*_mis, bins=5))
+close("ece/bins=1 is the mean gap over the whole range", ece_score(*_mis, bins=1), 0.49, tol=1e-9)
+
+# laya.evals.ece delegates here, so the CLI's gate inherits the refusal
+from laya.evals import ece as _evals_ece # noqa: E402
+
+try:
+ _evals_ece([0.99] * 10, [True] * 5 + [False] * 5, bins=0)
+ FAIL.append("ece/laya.evals.ece inherits the refusal: returned instead of raising")
+except ValueError as exc:
+ check_true("ece/laya.evals.ece inherits the refusal", _ECE_REFUSAL in str(exc), str(exc))
+
# =============================================================== confidence_from_probs
close("confidence/uniform over 2 is 0", confidence_from_probs(np.array([0.5, 0.5]), 2), 0.0)