Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): refuse a NaN --min-accuracy or --max-ece like the --min pairs
sameedkhan17d9b4040bd55a
diff --git a/laya/evals_cli.py b/laya/evals_cli.pyindex 7ab6ca16..66897dc1 100644--- a/laya/evals_cli.py+++ b/laya/evals_cli.py@@ -106,6 +106,18 @@ def _parse_pairs(pairs: Optional[Sequence[str]]) -> Dict[str, float]:return out+def _limit(raw: str) -> float:+ """argparse `type=` for `--min-accuracy` and `--max-ece`: refuse "nan" as `_parse_pairs` does."""+ try:+ number = float(raw)+ except ValueError:+ raise argparse.ArgumentTypeError("%r is not a number" % raw)+ # float() accepts "nan", and a NaN limit would disable the gate it names.+ if math.isnan(number):+ raise argparse.ArgumentTypeError("%r is not a number" % raw)+ return number++def _parse_revisions(pairs: Optional[Sequence[str]]) -> Tuple[Optional[str], Dict[str, str]]:"""Read `--revision` into the two forms `Router` takes: one commit, or one per checkpoint.@@ -193,9 +205,9 @@ def _build_parser() -> argparse.ArgumentParser:run.add_argument("--baseline", help="a baseline report JSON to compare against")run.add_argument("--tolerance", action="append", metavar="METRIC=VALUE",help="allowed absolute drift from the baseline; repeatable")- run.add_argument("--min-accuracy", type=float,+ run.add_argument("--min-accuracy", type=_limit,help="minimum accuracy (choice, else noul) for the whole dataset")- run.add_argument("--max-ece", type=float, help="maximum expected calibration error")+ run.add_argument("--max-ece", type=_limit, help="maximum expected calibration error")run.add_argument("--score-within", dest="score_within", type=float, action="append",metavar="TOL",help="also report score_within_TOL, the fraction of score answers within TOL "diff --git a/tests/test_evals.py b/tests/test_evals.pyindex 914b3a9b..59cbaea2 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -1395,12 +1395,22 @@ def test_a_nan_metric_fails_every_gate():assert healthy.compare({"overall": {"choice_accuracy": 0.9}})[0]-def test_cli_rejects_a_nan_limit_or_tolerance():+def test_cli_rejects_a_nan_limit_or_tolerance(capsys):from laya import evals_cliwith pytest.raises(EvalError, match="not a number"):evals_cli._parse_pairs(["choice_accuracy=nan"])+ # The two dedicated flags bypass `_parse_pairs`; a NaN there disabled the gate the same way.+ for flag in ("--min-accuracy", "--max-ece"):+ with pytest.raises(SystemExit) as exc:+ evals_cli.main(["run", "data.jsonl", flag, "nan"])+ assert exc.value.code == 2+ assert "not a number" in capsys.readouterr().err+ args = evals_cli._build_parser().parse_args(["run", "data.jsonl", "--min-accuracy", "0.8",+ "--max-ece", "0.1"])+ assert (args.min_accuracy, args.max_ece) == (0.8, 0.1)+def test_default_evaluators_cover_the_three_types():names = {e.name for e in default_evaluators()}