Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): fail the gates on a NaN metric, limit or tolerance

abhijithneilabraham4c2c688310b3
diff --git a/laya/evals.py b/laya/evals.py
index 878a27f4..33a6f34b 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -430,7 +430,8 @@ class EvalReport:
absolute difference allowed for that metric. A baseline metric the report no longer has
fails the comparison (its delta carries ``missing: True`` and a NaN value): a run whose
every example errored under ``on_error="skip"`` has an empty ``overall``, and must not
- pass the gate by having nothing left to compare.
+ pass the gate by having nothing left to compare. A NaN metric, baseline or tolerance
+ fails the comparison too.
"""
base = (baseline or {}).get("overall", baseline or {})
tolerances = tolerances or {}
@@ -451,7 +452,9 @@ class EvalReport:
diff = value - float(base_value)
deltas[metric] = {"baseline": float(base_value), "value": value,
"diff": diff, "tolerance": allowed}
- if abs(diff) > allowed:
+ # `not ... <=` rather than `>`: every comparison with NaN is False, so a NaN on
+ # either side would otherwise pass as "did not move".
+ if not abs(diff) <= allowed:
ok = False
return ok, deltas
diff --git a/laya/evals_cli.py b/laya/evals_cli.py
index 4139570a..67328e18 100644
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@ -96,9 +96,13 @@ def _parse_pairs(pairs: Optional[Sequence[str]]) -> Dict[str, float]:
if not name or not raw:
raise EvalError("expected NAME=VALUE, got %r" % pair)
try:
- out[name.strip()] = float(raw)
+ number = float(raw)
except ValueError:
raise EvalError("%r is not a number in %r" % (raw, pair))
+ # float() accepts "nan", and a NaN limit or tolerance would disable the gate it names.
+ if math.isnan(number):
+ raise EvalError("%r is not a number in %r" % (raw, pair))
+ out[name.strip()] = number
return out
@@ -232,12 +236,16 @@ def _check_thresholds(overall: Dict[str, float], mins: Dict[str, float],
value = overall.get(name)
if value is None:
failures.append("metric %r is not in the report" % name)
+ elif math.isnan(value):
+ failures.append("metric %r is NaN" % name)
elif value < limit:
failures.append("%s=%.4f is below the minimum %.4f" % (name, value, limit))
for name, limit in maxs.items():
value = overall.get(name)
if value is None:
failures.append("metric %r is not in the report" % name)
+ elif math.isnan(value):
+ failures.append("metric %r is NaN" % name)
elif value > limit:
failures.append("%s=%.4f is above the maximum %.4f" % (name, value, limit))
return failures