Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
test(evals): a NaN metric, baseline or tolerance fails the gates
abhijithneilabraham39ff9ae870a2
diff --git a/tests/test_evals.py b/tests/test_evals.pyindex 249060ec..bc29ebff 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -904,6 +904,34 @@ def test_compare_fails_when_a_baseline_metric_is_missing():assert "missing" not in deltas["choice_accuracy"] and deltas["noul_accuracy"]["missing"]+def test_a_nan_metric_fails_every_gate():+ """Every comparison with NaN is False, so a NaN metric used to pass --min, --max and the+ baseline comparison alike."""+ from laya import evals_cli++ nan = float("nan")+ report = EvalReport(overall={"score_mae": nan, "mean_confidence": nan})+ failures = evals_cli._check_thresholds(report.overall, {"mean_confidence": 0.9}, {"score_mae": 0.1})+ assert len(failures) == 2 and all("NaN" in failure for failure in failures)++ ok, deltas = report.compare({"overall": {"score_mae": 0.05, "mean_confidence": 0.95}},+ {"score_mae": 0.1, "mean_confidence": 0.1})+ assert not ok and set(deltas) == {"score_mae", "mean_confidence"}++ # A NaN in the baseline, or as the tolerance, is the same case from the other side.+ healthy = EvalReport(overall={"choice_accuracy": 0.9})+ assert not healthy.compare({"overall": {"choice_accuracy": nan}}, {"choice_accuracy": 1.0})[0]+ assert not healthy.compare({"overall": {"choice_accuracy": 0.9}}, {"choice_accuracy": nan})[0]+ assert healthy.compare({"overall": {"choice_accuracy": 0.9}})[0]+++def test_cli_rejects_a_nan_limit_or_tolerance():+ from laya import evals_cli++ with pytest.raises(EvalError, match="not a number"):+ evals_cli._parse_pairs(["choice_accuracy=nan"])++def test_default_evaluators_cover_the_three_types():names = {e.name for e in default_evaluators()}assert {"choice_accuracy", "noul_accuracy", "score_mae", "mean_confidence"} <= names