Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): fail the gates on a NaN metric, limit or tolerance
abhijithneilabraham4c2c688310b3
diff --git a/laya/evals.py b/laya/evals.pyindex 878a27f4..33a6f34b 100644--- a/laya/evals.py+++ b/laya/evals.py@@ -430,7 +430,8 @@ class EvalReport:absolute difference allowed for that metric. A baseline metric the report no longer hasfails the comparison (its delta carries ``missing: True`` and a NaN value): a run whoseevery example errored under ``on_error="skip"`` has an empty ``overall``, and must not- pass the gate by having nothing left to compare.+ pass the gate by having nothing left to compare. A NaN metric, baseline or tolerance+ fails the comparison too."""base = (baseline or {}).get("overall", baseline or {})tolerances = tolerances or {}@@ -451,7 +452,9 @@ class EvalReport:diff = value - float(base_value)deltas[metric] = {"baseline": float(base_value), "value": value,"diff": diff, "tolerance": allowed}- if abs(diff) > allowed:+ # `not ... <=` rather than `>`: every comparison with NaN is False, so a NaN on+ # either side would otherwise pass as "did not move".+ if not abs(diff) <= allowed:ok = Falsereturn ok, deltasdiff --git a/laya/evals_cli.py b/laya/evals_cli.pyindex 4139570a..67328e18 100644--- a/laya/evals_cli.py+++ b/laya/evals_cli.py@@ -96,9 +96,13 @@ def _parse_pairs(pairs: Optional[Sequence[str]]) -> Dict[str, float]:if not name or not raw:raise EvalError("expected NAME=VALUE, got %r" % pair)try:- out[name.strip()] = float(raw)+ number = float(raw)except ValueError:raise EvalError("%r is not a number in %r" % (raw, pair))+ # float() accepts "nan", and a NaN limit or tolerance would disable the gate it names.+ if math.isnan(number):+ raise EvalError("%r is not a number in %r" % (raw, pair))+ out[name.strip()] = numberreturn out@@ -232,12 +236,16 @@ def _check_thresholds(overall: Dict[str, float], mins: Dict[str, float],value = overall.get(name)if value is None:failures.append("metric %r is not in the report" % name)+ elif math.isnan(value):+ failures.append("metric %r is NaN" % name)elif value < limit:failures.append("%s=%.4f is below the minimum %.4f" % (name, value, limit))for name, limit in maxs.items():value = overall.get(name)if value is None:failures.append("metric %r is not in the report" % name)+ elif math.isnan(value):+ failures.append("metric %r is NaN" % name)elif value > limit:failures.append("%s=%.4f is above the maximum %.4f" % (name, value, limit))return failures