Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): refuse a NaN --min-accuracy or --max-ece like the --min pairs

sameedkhan17d9b4040bd55a
diff --git a/laya/evals_cli.py b/laya/evals_cli.py
index 7ab6ca16..66897dc1 100644
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@ -106,6 +106,18 @@ def _parse_pairs(pairs: Optional[Sequence[str]]) -> Dict[str, float]:
return out
+def _limit(raw: str) -> float:
+ """argparse `type=` for `--min-accuracy` and `--max-ece`: refuse "nan" as `_parse_pairs` does."""
+ try:
+ number = float(raw)
+ except ValueError:
+ raise argparse.ArgumentTypeError("%r is not a number" % raw)
+ # float() accepts "nan", and a NaN limit would disable the gate it names.
+ if math.isnan(number):
+ raise argparse.ArgumentTypeError("%r is not a number" % raw)
+ return number
+
+
def _parse_revisions(pairs: Optional[Sequence[str]]) -> Tuple[Optional[str], Dict[str, str]]:
"""Read `--revision` into the two forms `Router` takes: one commit, or one per checkpoint.
@@ -193,9 +205,9 @@ def _build_parser() -> argparse.ArgumentParser:
run.add_argument("--baseline", help="a baseline report JSON to compare against")
run.add_argument("--tolerance", action="append", metavar="METRIC=VALUE",
help="allowed absolute drift from the baseline; repeatable")
- run.add_argument("--min-accuracy", type=float,
+ run.add_argument("--min-accuracy", type=_limit,
help="minimum accuracy (choice, else noul) for the whole dataset")
- run.add_argument("--max-ece", type=float, help="maximum expected calibration error")
+ run.add_argument("--max-ece", type=_limit, help="maximum expected calibration error")
run.add_argument("--score-within", dest="score_within", type=float, action="append",
metavar="TOL",
help="also report score_within_TOL, the fraction of score answers within TOL "
diff --git a/tests/test_evals.py b/tests/test_evals.py
index 914b3a9b..59cbaea2 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -1395,12 +1395,22 @@ def test_a_nan_metric_fails_every_gate():
assert healthy.compare({"overall": {"choice_accuracy": 0.9}})[0]
-def test_cli_rejects_a_nan_limit_or_tolerance():
+def test_cli_rejects_a_nan_limit_or_tolerance(capsys):
from laya import evals_cli
with pytest.raises(EvalError, match="not a number"):
evals_cli._parse_pairs(["choice_accuracy=nan"])
+ # The two dedicated flags bypass `_parse_pairs`; a NaN there disabled the gate the same way.
+ for flag in ("--min-accuracy", "--max-ece"):
+ with pytest.raises(SystemExit) as exc:
+ evals_cli.main(["run", "data.jsonl", flag, "nan"])
+ assert exc.value.code == 2
+ assert "not a number" in capsys.readouterr().err
+ args = evals_cli._build_parser().parse_args(["run", "data.jsonl", "--min-accuracy", "0.8",
+ "--max-ece", "0.1"])
+ assert (args.min_accuracy, args.max_ece) == (0.8, 0.1)
+
def test_default_evaluators_cover_the_three_types():
names = {e.name for e in default_evaluators()}