Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(calibrate): read bins and min_bucket_n as the module's own positive integers

Aashish4f76fb6c1d18
diff --git a/laya/calibrate.py b/laya/calibrate.py
index dc977e09..73b0d483 100644
--- a/laya/calibrate.py
+++ b/laya/calibrate.py
@@ -92,6 +92,27 @@ def _validated_pair(logits, target, k=None):
return logits, target, int(k)
+def _positive_int(name: str, value: int) -> int:
+ """A count parameter: an integer >= 1, normalised to a Python `int`, or a `ValueError` naming it.
+
+ This is the rule the module already applies everywhere a count is checked -- `min_n` in
+ :func:`fit_one_temperature`, `k` in :func:`_validated_pair` (which returns `int(k)` for the same
+ reason), and the `"bins"` of every `binning_map` entry :func:`apply_calibration_payload` reads --
+ applied to the two fitting parameters that were checked with less. The asymmetry was not
+ cosmetic: a `bool` is an `Integral`, so `bins` computed as a comparison (`len(bucket) > 15`)
+ passed the old `bins < 1` guard and emitted `{"bins": True, ...}`, a map the loader refuses on
+ its boolean arm. And a `numpy` integer is an `Integral` but not a Python `int`, so a `bins`
+ derived from an array shape emitted a map `Agent.save_calibration`'s own `json.dump` cannot write
+ -- a fit that produced an artifact the file format rejects. `int()` keeps that second case
+ working. A float, a string or `None` is a mistake, and now names itself instead of raising
+ `TypeError` from `range()` (or, for a `min_bucket_n` that compares `nan` to a length, silently
+ disabling the sample floor the parameter exists to enforce).
+ """
+ if isinstance(value, bool) or not isinstance(value, Integral) or value < 1:
+ raise ValueError("%s must be a positive integer, got %r" % (name, value))
+ return int(value)
+
+
def _pairs_by_width(pairs: Sequence) -> List[Tuple[torch.Tensor, torch.Tensor]]:
"""One (logits, target) matrix per distinct option count, narrowest first.
@@ -401,6 +422,7 @@ def fit_abstention_thresholds(records: Iterable, temperature: Sequence[float],
"""
if not 0.0 <= target_error <= 1.0:
raise ValueError("target_error must be in [0.0, 1.0], got %r" % (target_error,))
+ min_bucket_n = _positive_int("min_bucket_n", min_bucket_n)
recs = _iter_records(records)
by_bucket: Dict[str, List[Tuple[float, int]]] = {}
for qt, z, t, k in recs:
@@ -455,8 +477,8 @@ def fit_binning_map(records: Iterable, temperature: Sequence[float],
fabricated 0, not that it passes through -- only a bucket the map has no entry for does that
(:func:`apply_binning_map`).
"""
- if bins < 1:
- raise ValueError("bins must be >= 1, got %r" % (bins,))
+ bins = _positive_int("bins", bins)
+ min_bucket_n = _positive_int("min_bucket_n", min_bucket_n)
recs = _iter_records(records)
by_bucket: Dict[str, List[Tuple[float, int]]] = {}
for qt, z, t, k in recs:
diff --git a/tests/test_calibrate.py b/tests/test_calibrate.py
index 7f119feb..d9ca375b 100644
--- a/tests/test_calibrate.py
+++ b/tests/test_calibrate.py
@@ -1173,6 +1173,61 @@ try:
except ValueError:
check_true("binning/bins must be >= 1", True)
+# `bins` and `min_bucket_n` are counts, and this module already has a count rule -- `min_n` in
+# `fit_one_temperature`, `k` in `_validated_pair`, and the `"bins"` of every `binning_map` entry a
+# calibration payload carries. Only `bins < 1` was applied to these two, so everything below either
+# escaped the guard as a `TypeError` from `range()` / a numpy cast, or -- the arms that were
+# *accepted* -- came back as a map. Checked message and all, because "raised something" was exactly
+# what was wrong.
+for _name, _bad in [("bins", 0), ("bins", -3), ("bins", 2.5), ("bins", 10.0), ("bins", float("nan")),
+ ("bins", float("inf")), ("bins", True), ("bins", False), ("bins", None),
+ ("bins", "10"), ("min_bucket_n", 0), ("min_bucket_n", -3),
+ ("min_bucket_n", 2.5), ("min_bucket_n", float("nan")), ("min_bucket_n", True),
+ ("min_bucket_n", None)]:
+ try:
+ fit_binning_map(mixed, fitted["temperature"], fitted["temperature_by_options"],
+ **{_name: _bad})
+ _res = "no ValueError"
+ except ValueError as _exc:
+ _res = str(_exc)
+ except Exception as _exc:
+ _res = "raised %s" % type(_exc).__name__
+ check("binning/%s refuses %r" % (_name, _bad), _res,
+ "%s must be a positive integer, got %r" % (_name, _bad))
+
+# The two arms the old guard let through, and what they cost. `numpy.int64` is an `Integral`, so it
+# passed `bins < 1`, fitted happily, and put a numpy scalar in `"bins"` that `save_calibration`'s
+# `json.dump` refuses to serialise. `True` is an `Integral` too: one bin, and a map
+# `apply_calibration_payload` rejects on its boolean arm -- a producer whose own file format cannot
+# read what it emits.
+_np_map = fit_binning_map(mixed, fitted["temperature"], fitted["temperature_by_options"],
+ bins=np.int64(12))
+check("binning/numpy bins arrives as a Python int",
+ type(next(iter(_np_map.values()))["bins"]).__name__, "int")
+check("binning/numpy bins keeps every requested bin",
+ len(next(iter(_np_map.values()))["values"]), 12)
+_np_payload = calibration_payload(fitted["temperature"], fitted["temperature_by_options"],
+ binning_map=_np_map)
+try:
+ _np_round = json.loads(json.dumps(_np_payload))
+except TypeError as _exc:
+ _np_round = "json %s" % _exc
+check_true("binning/numpy bins payload writes as JSON", isinstance(_np_round, dict), _np_round)
+if isinstance(_np_round, dict):
+ # Only a payload that made it to JSON can be read back, so a regression here fails the check
+ # above by name instead of aborting the suite on the loader's own type error.
+ _np_stub = type("Stub", (), {})()
+ apply_calibration_payload(_np_stub, _np_round)
+ check("binning/numpy bins reloads as the map that was fitted", _np_stub.binning_map, _np_map)
+
+# A floor given as a numpy integer is still a floor: the guard is this module's `Integral` rule, not
+# a narrower one, so a `min_bucket_n` that came out of an array expression keeps working.
+check("binning/numpy floor reads as the int floor",
+ fit_binning_map(mixed, fitted["temperature"], fitted["temperature_by_options"],
+ bins=12, min_bucket_n=np.int64(MIN_BINNING_BUCKET_N)),
+ fit_binning_map(mixed, fitted["temperature"], fitted["temperature_by_options"],
+ bins=12, min_bucket_n=MIN_BINNING_BUCKET_N))
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
diff --git a/tests/test_conformal_abstention.py b/tests/test_conformal_abstention.py
index a5cb11c0..6904044f 100644
--- a/tests/test_conformal_abstention.py
+++ b/tests/test_conformal_abstention.py
@@ -245,6 +245,30 @@ check("validate/the same records at floor 1 fit every bucket they touch",
sorted(_all_buckets), ["choice:11+", "choice:2"])
check("validate/and pass the validator unchanged", _accepts(dict(_all_buckets)), dict(_all_buckets))
+# The floor had no guard, and a comparison against a length is where a bad one goes unnoticed:
+# `len(pairs) < nan` is False for every bucket, and so is `< -1`, so the sentence above
+# ("`min_bucket_n` omits buckets too small to fit") was only ever true of a floor that happened to be
+# an integer. A `nan` or `2.5` arrived quietly -- a floor from a plain `/` division, say -- and the
+# gate was fitted on whatever records the caller had. Same positive-integer rule `fit_binning_map`
+# applies, checked by message because the old behaviour was silence rather than a wrong exception.
+for _bad in (0, -3, 2.5, float("nan"), float("inf"), True, None, "100"):
+ try:
+ fit_abstention_thresholds(_small, [1.0, 1.0, 1.0], {}, target_error=0.10, min_bucket_n=_bad)
+ _res = "no ValueError"
+ except ValueError as _exc:
+ _res = str(_exc)
+ except Exception as _exc:
+ _res = "raised %s" % type(_exc).__name__
+ check("floor/%r is refused by name" % (_bad,), _res,
+ "min_bucket_n must be a positive integer, got %r" % (_bad,))
+
+# And the rule did not get stricter than the module's own: an integer floor below the shipped one
+# still fits, which is how a caller asks for a bucket the default would have dropped.
+check("floor/an integer below the shipped floor still fits",
+ sorted(fit_abstention_thresholds(_small, [1.0, 1.0, 1.0], {}, target_error=0.10,
+ min_bucket_n=1)),
+ ["choice:3-5"])
+
BOGUS = {
"choice:2-5": "an invented band",
"choice:12": "an option count, not a band",