Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): refuse a non-finite --min/--max/--tolerance gate parameter

Aashishb866a34e1df8
diff --git a/laya/evals_cli.py b/laya/evals_cli.py
index 55732f3c..270b965d 100644
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@ -99,21 +99,26 @@ def _parse_pairs(pairs: Optional[Sequence[str]]) -> Dict[str, float]:
number = float(raw)
except ValueError:
raise EvalError("%r is not a number in %r" % (raw, pair))
- # float() accepts "nan", and a NaN limit or tolerance would disable the gate it names.
- if math.isnan(number):
+ # A non-finite limit or tolerance disables the gate it names rather than tightening it:
+ # `--max ece=inf` and `--min accuracy=-inf` pass every measurement, and
+ # `--tolerance METRIC=inf` makes `compare` accept a drift of any size and still exit 0 --
+ # the code that means "the gate ran and the numbers passed". `nan` was refused for this
+ # exact reason; `inf` reaches the same gate by a route `nan` has no equivalent of, because
+ # `float("1e999")` overflows to `inf` and raises nothing.
+ if not math.isfinite(number):
raise EvalError("%r is not a number in %r" % (raw, pair))
out[name.strip()] = number
return out
def _limit(raw: str) -> float:
- """argparse `type=` for `--min-accuracy` and `--max-ece`: refuse "nan" as `_parse_pairs` does."""
+ """argparse `type=` for `--min-accuracy` and `--max-ece`: refuse a non-finite number."""
try:
number = float(raw)
except ValueError:
raise argparse.ArgumentTypeError("%r is not a number" % raw)
- # float() accepts "nan", and a NaN limit would disable the gate it names.
- if math.isnan(number):
+ # Same gate as `_parse_pairs`: a non-finite limit decides nothing.
+ if not math.isfinite(number):
raise argparse.ArgumentTypeError("%r is not a number" % raw)
return number
diff --git a/tests/test_evals.py b/tests/test_evals.py
index cc469c2e..03e44b28 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -1704,6 +1704,57 @@ def test_cli_rejects_a_nan_limit_or_tolerance(capsys):
assert (args.min_accuracy, args.max_ece) == (0.8, 0.1)
+def test_cli_rejects_an_infinite_limit_or_tolerance(capsys):
+ """`inf` disables a gate exactly as `nan` does, and arrives without anyone typing "inf"."""
+ from laya import evals_cli
+
+ # Every spelling `float()` answers with a non-finite number, plus the overflow that is the
+ # reason this is not only a typo case: `--max ece=1e999` parses clean and means no cap.
+ for raw in ("inf", "-inf", "+inf", "Infinity", "-Infinity", "1e999", "inf ", " INF"):
+ with pytest.raises(EvalError, match="is not a number") as exc:
+ evals_cli._parse_pairs(["ece=%s" % raw])
+ assert raw in str(exc.value), "the message names the value the caller typed"
+
+ # The two dedicated flags bypass `_parse_pairs`, and exit 2 rather than a traceback.
+ for flag in ("--min-accuracy", "--max-ece"):
+ for raw in ("inf", "1e999"):
+ with pytest.raises(SystemExit) as exc:
+ evals_cli.main(["run", "data.jsonl", flag, raw])
+ assert exc.value.code == 2
+ assert "not a number" in capsys.readouterr().err
+ # A leading "-" is argparse's own refusal here, not `_limit`'s: the value reads as the
+ # next option. Still exit 2, and `--min accuracy=-inf` -- the form that reaches
+ # `_parse_pairs` -- is refused above.
+ with pytest.raises(SystemExit) as exc:
+ evals_cli.main(["run", "data.jsonl", flag, "-inf"])
+ assert exc.value.code == 2
+ assert "expected one argument" in capsys.readouterr().err
+
+ # Only the two non-finite kinds are refused: a boundary number is still a limit.
+ assert evals_cli._parse_pairs(["ece=0.0", "choice_accuracy=1.0", "brier=-0.0"]) == {
+ "ece": 0.0, "choice_accuracy": 1.0, "brier": 0.0}
+ assert evals_cli._limit("1e308") == 1e308
+
+
+def test_an_infinite_limit_would_have_decided_nothing():
+ """The harm the guard above prevents, pinned so relaxing it cannot go quiet.
+
+ Without the refusal these are the values `_check_thresholds` and `compare` compute from a
+ non-finite limit, and each one is a pass: `laya-evals` would exit 0 having gated nothing. A
+ test that asserts them is not asserting correct behaviour -- it is the witness that the
+ guard's absence was a defect.
+ """
+ from laya import evals_cli
+
+ inf = float("inf")
+ assert evals_cli._check_thresholds({"ece": 0.42}, {}, {"ece": inf}) == []
+ assert evals_cli._check_thresholds({"choice_accuracy": 0.10}, {"choice_accuracy": -inf}, {}) == []
+ ok, deltas = EvalReport(config={}, overall={"choice_accuracy": 0.10}).compare(
+ {"overall": {"choice_accuracy": 0.90}}, {"choice_accuracy": inf})
+ assert ok, "an 0.80 drop passed a gate whose tolerance was inf"
+ assert deltas["choice_accuracy"]["diff"] == pytest.approx(-0.8)
+
+
def test_default_evaluators_cover_the_three_types():
names = {e.name for e in default_evaluators()}
assert {"choice_accuracy", "noul_accuracy", "score_mae", "mean_confidence"} <= names