Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): name the delta the gate failed on, including NaN and incomparable ones

PerryLink70293cd478a0
diff --git a/laya/evals.py b/laya/evals.py
index acb4dab7..f0b1013a 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -1389,7 +1389,14 @@ def assert_regression(report: EvalReport, baseline: Dict[str, Any],
"""Raise AssertionError when `report` drifts from `baseline` beyond `tolerances`."""
ok, deltas = report.compare(baseline, tolerances)
if not ok:
+ # `compare()` fails three shapes, and only the numeric one makes `abs(diff) > tolerance`
+ # true: a missing metric and an incomparable coverage definition both store a NaN diff
+ # (laya/evals.py:928-936), and every comparison with NaN is False, which is exactly why
+ # compare() tests `not abs(diff) <= allowed` (comment at laya/evals.py:946-949). Filtering
+ # on `>` alone dropped both, so the gate raised "evaluation regressed: {}" and lost the
+ # metric and the reason compare() had already computed.
failed = {m: d for m, d in deltas.items()
- if d.get("missing") or abs(d["diff"]) > d["tolerance"]}
+ if d.get("missing") or d.get("incomparable")
+ or not abs(d["diff"]) <= d["tolerance"]}
raise AssertionError("evaluation regressed: %s" % json.dumps(failed, sort_keys=True))
return deltas
diff --git a/tests/test_evals.py b/tests/test_evals.py
index f99bf615..5fe93193 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -3935,3 +3935,22 @@ def test_chunking_folds_criteria_key_order_on_the_request_form():
evaluators=[ChoiceAccuracy()], batch_size=2)
assert listed_report.config["timing"]["rows_grouped"] == 0, \
"a reordered list criteria stays positional even for the request form"
+
+
+
+def test_assert_regression_names_incomparable_and_nan_deltas():
+ """The gate must name every delta compare() failed, not just the numeric ones.
+
+ An incomparable coverage definition and a NaN tolerance both fail compare() without
+ `abs(diff) > tolerance` holding, so filtering the message on that comparison alone left
+ `evaluation regressed: {}` with neither the metric nor the reason.
+ """
+ report = EvalReport(config={"coverage_metric_definition": 2}, overall={"aurc": 0.2})
+ baseline = {"config": {"coverage_metric_definition": 1}, "overall": {"aurc": 0.1}}
+ with pytest.raises(AssertionError, match="aurc"):
+ assert_regression(report, baseline)
+
+ with pytest.raises(AssertionError, match="choice_accuracy"):
+ assert_regression(EvalReport(overall={"choice_accuracy": 0.8}),
+ {"overall": {"choice_accuracy": 0.8}},
+ {"choice_accuracy": float("nan")})