Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): refuse a non-finite --min/--max/--tolerance gate parameter
Aashishb866a34e1df8
diff --git a/laya/evals_cli.py b/laya/evals_cli.pyindex 55732f3c..270b965d 100644--- a/laya/evals_cli.py+++ b/laya/evals_cli.py@@ -99,21 +99,26 @@ def _parse_pairs(pairs: Optional[Sequence[str]]) -> Dict[str, float]:number = float(raw)except ValueError:raise EvalError("%r is not a number in %r" % (raw, pair))- # float() accepts "nan", and a NaN limit or tolerance would disable the gate it names.- if math.isnan(number):+ # A non-finite limit or tolerance disables the gate it names rather than tightening it:+ # `--max ece=inf` and `--min accuracy=-inf` pass every measurement, and+ # `--tolerance METRIC=inf` makes `compare` accept a drift of any size and still exit 0 --+ # the code that means "the gate ran and the numbers passed". `nan` was refused for this+ # exact reason; `inf` reaches the same gate by a route `nan` has no equivalent of, because+ # `float("1e999")` overflows to `inf` and raises nothing.+ if not math.isfinite(number):raise EvalError("%r is not a number in %r" % (raw, pair))out[name.strip()] = numberreturn outdef _limit(raw: str) -> float:- """argparse `type=` for `--min-accuracy` and `--max-ece`: refuse "nan" as `_parse_pairs` does."""+ """argparse `type=` for `--min-accuracy` and `--max-ece`: refuse a non-finite number."""try:number = float(raw)except ValueError:raise argparse.ArgumentTypeError("%r is not a number" % raw)- # float() accepts "nan", and a NaN limit would disable the gate it names.- if math.isnan(number):+ # Same gate as `_parse_pairs`: a non-finite limit decides nothing.+ if not math.isfinite(number):raise argparse.ArgumentTypeError("%r is not a number" % raw)return numberdiff --git a/tests/test_evals.py b/tests/test_evals.pyindex cc469c2e..03e44b28 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -1704,6 +1704,57 @@ def test_cli_rejects_a_nan_limit_or_tolerance(capsys):assert (args.min_accuracy, args.max_ece) == (0.8, 0.1)+def test_cli_rejects_an_infinite_limit_or_tolerance(capsys):+ """`inf` disables a gate exactly as `nan` does, and arrives without anyone typing "inf"."""+ from laya import evals_cli++ # Every spelling `float()` answers with a non-finite number, plus the overflow that is the+ # reason this is not only a typo case: `--max ece=1e999` parses clean and means no cap.+ for raw in ("inf", "-inf", "+inf", "Infinity", "-Infinity", "1e999", "inf ", " INF"):+ with pytest.raises(EvalError, match="is not a number") as exc:+ evals_cli._parse_pairs(["ece=%s" % raw])+ assert raw in str(exc.value), "the message names the value the caller typed"++ # The two dedicated flags bypass `_parse_pairs`, and exit 2 rather than a traceback.+ for flag in ("--min-accuracy", "--max-ece"):+ for raw in ("inf", "1e999"):+ with pytest.raises(SystemExit) as exc:+ evals_cli.main(["run", "data.jsonl", flag, raw])+ assert exc.value.code == 2+ assert "not a number" in capsys.readouterr().err+ # A leading "-" is argparse's own refusal here, not `_limit`'s: the value reads as the+ # next option. Still exit 2, and `--min accuracy=-inf` -- the form that reaches+ # `_parse_pairs` -- is refused above.+ with pytest.raises(SystemExit) as exc:+ evals_cli.main(["run", "data.jsonl", flag, "-inf"])+ assert exc.value.code == 2+ assert "expected one argument" in capsys.readouterr().err++ # Only the two non-finite kinds are refused: a boundary number is still a limit.+ assert evals_cli._parse_pairs(["ece=0.0", "choice_accuracy=1.0", "brier=-0.0"]) == {+ "ece": 0.0, "choice_accuracy": 1.0, "brier": 0.0}+ assert evals_cli._limit("1e308") == 1e308+++def test_an_infinite_limit_would_have_decided_nothing():+ """The harm the guard above prevents, pinned so relaxing it cannot go quiet.++ Without the refusal these are the values `_check_thresholds` and `compare` compute from a+ non-finite limit, and each one is a pass: `laya-evals` would exit 0 having gated nothing. A+ test that asserts them is not asserting correct behaviour -- it is the witness that the+ guard's absence was a defect.+ """+ from laya import evals_cli++ inf = float("inf")+ assert evals_cli._check_thresholds({"ece": 0.42}, {}, {"ece": inf}) == []+ assert evals_cli._check_thresholds({"choice_accuracy": 0.10}, {"choice_accuracy": -inf}, {}) == []+ ok, deltas = EvalReport(config={}, overall={"choice_accuracy": 0.10}).compare(+ {"overall": {"choice_accuracy": 0.90}}, {"choice_accuracy": inf})+ assert ok, "an 0.80 drop passed a gate whose tolerance was inf"+ assert deltas["choice_accuracy"]["diff"] == pytest.approx(-0.8)++def test_default_evaluators_cover_the_three_types():names = {e.name for e in default_evaluators()}assert {"choice_accuracy", "noul_accuracy", "score_mae", "mean_confidence"} <= names