Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(calibrate): refuse a boolean target_error in fit_abstention_thresholds

aashish254f67d41afa32d
diff --git a/laya/calibrate.py b/laya/calibrate.py
index 73b0d483..3d35207e 100644
--- a/laya/calibrate.py
+++ b/laya/calibrate.py
@@ -420,7 +420,11 @@ def fit_abstention_thresholds(records: Iterable, temperature: Sequence[float],
coverage on un-binned confidences, and admits 94.5% of answers at 25.6% error once the same
number is compared against binned ones.
"""
- if not 0.0 <= target_error <= 1.0:
+ # A bool is the one value a range test cannot see: `0.0 <= True <= 1.0` holds, so `False` would
+ # be read as 0.0 -- the cut that abstains on every answer -- and `True` as 1.0, which gates
+ # nothing. Both fit a map, so nothing downstream would mention it. `TrainConfig` refuses the
+ # same bool on the way to this call (`train.py`), as does `_iter_records` for a `qtype`.
+ if isinstance(target_error, bool) or not 0.0 <= target_error <= 1.0:
raise ValueError("target_error must be in [0.0, 1.0], got %r" % (target_error,))
min_bucket_n = _positive_int("min_bucket_n", min_bucket_n)
recs = _iter_records(records)
diff --git a/tests/test_conformal_abstention.py b/tests/test_conformal_abstention.py
index 6904044f..4dd15907 100644
--- a/tests/test_conformal_abstention.py
+++ b/tests/test_conformal_abstention.py
@@ -269,6 +269,35 @@ check("floor/an integer below the shipped floor still fits",
min_bucket_n=1)),
["choice:3-5"])
+# `target_error` is a rate, and a bool is the one value the range test cannot see: `0.0 <= True`
+# holds, so `False` was read as 0.0 and `True` as 1.0 -- the two failures this fit exists to keep
+# apart. Measured on the 480 `records` above at floor 50: 0.0 fits {'choice:2': 1.0,
+# 'choice:11+': 1.0}, which accepts 0 of 480 answers; 0.10 fits {'choice:2': 0.97,
+# 'choice:11+': 0.55}, which accepts 240 of them with 0 errors among the accepted; 1.0 fits
+# {'choice:2': 0.85, 'choice:11+': 0.33}, which accepts all 480 at 156 errors (32.5%). Either map is
+# something `check_min_confidence` accepts and `laya.train` writes into a checkpoint, so the caller
+# never heard about the coercion. `TrainConfig` refuses the same bool one call earlier, on its way
+# into this function; this is that arm, on the public surface.
+for _bad, _coerces in ((True, 1.0), (False, 0.0)):
+ try:
+ fit_abstention_thresholds(_small, [1.0, 1.0, 1.0], {}, target_error=_bad, min_bucket_n=1)
+ _res = "no ValueError (read as %r)" % (_coerces,)
+ except ValueError as _exc:
+ _res = str(_exc)
+ except Exception as _exc:
+ _res = "raised %s" % type(_exc).__name__
+ check("target_error/%r is refused by name" % (_bad,), _res,
+ "target_error must be in [0.0, 1.0], got %r" % (_bad,))
+
+# And the arm did not get stricter than the two ends it exists to protect: refusing 0.0 and 1.0
+# would refuse a caller genuinely asking for no tolerance, and those two are what the bools meant.
+check("target_error/0.0 still fits the abstain-on-everything cut",
+ fit_abstention_thresholds(_small, [1.0, 1.0, 1.0], {}, target_error=0.0, min_bucket_n=1),
+ {"choice:3-5": 1.0})
+check("target_error/1.0 still fits the ungated cut",
+ fit_abstention_thresholds(_small, [1.0, 1.0, 1.0], {}, target_error=1.0, min_bucket_n=1),
+ {"choice:3-5": 0.9})
+
BOGUS = {
"choice:2-5": "an invented band",
"choice:12": "an option count, not a band",