Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): fail the baseline comparison when a metric is missing

can-gaa-hou605bebb2a746
diff --git a/laya/evals.py b/laya/evals.py
index cac581d6..7c3e91bb 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -225,22 +225,28 @@ class EvalReport:
"""Compare `overall` to a baseline report's `overall`. Returns (ok, deltas).
With no `tolerances`, every shared metric must match exactly; a tolerance is the maximum
- absolute difference allowed for that metric.
+ absolute difference allowed for that metric. A baseline metric the report no longer has
+ fails the comparison (its delta carries ``missing: True`` and a NaN value): a run whose
+ every example errored under ``on_error="skip"`` has an empty ``overall``, and must not
+ pass the gate by having nothing left to compare.
"""
base = (baseline or {}).get("overall", baseline or {})
tolerances = tolerances or {}
deltas: Dict[str, Dict[str, float]] = {}
ok = True
for metric, base_value in base.items():
- if metric not in self.overall:
- continue
# Latency is informational; a re-run differs by timing noise, not quality, so it is
# compared only when a tolerance explicitly names it.
if metric.endswith("_ms") and metric not in tolerances:
continue
+ allowed = float(tolerances.get(metric, 0.0))
+ if metric not in self.overall:
+ deltas[metric] = {"baseline": float(base_value), "value": float("nan"),
+ "diff": float("nan"), "tolerance": allowed, "missing": True}
+ ok = False
+ continue
value = self.overall[metric]
diff = value - float(base_value)
- allowed = float(tolerances.get(metric, 0.0))
deltas[metric] = {"baseline": float(base_value), "value": value,
"diff": diff, "tolerance": allowed}
if abs(diff) > allowed:
@@ -366,6 +372,7 @@ def assert_regression(report: EvalReport, baseline: Dict[str, Any],
"""Raise AssertionError when `report` drifts from `baseline` beyond `tolerances`."""
ok, deltas = report.compare(baseline, tolerances)
if not ok:
- failed = {m: d for m, d in deltas.items() if abs(d["diff"]) > d["tolerance"]}
+ failed = {m: d for m, d in deltas.items()
+ if d.get("missing") or abs(d["diff"]) > d["tolerance"]}
raise AssertionError("evaluation regressed: %s" % json.dumps(failed, sort_keys=True))
return deltas
diff --git a/laya/evals_cli.py b/laya/evals_cli.py
index e656eba6..1dbd44a6 100644
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@ -108,6 +108,15 @@ def _check_thresholds(overall: Dict[str, float], mins: Dict[str, float],
return failures
+def _print_deltas(deltas: Dict[str, Dict[str, Any]]) -> None:
+ for metric, delta in sorted(deltas.items()):
+ if delta.get("missing"):
+ print("%-18s baseline=%.4f missing from the report" % (metric, delta["baseline"]))
+ continue
+ print("%-18s baseline=%.4f value=%.4f diff=%+.4f (tol %.4f)"
+ % (metric, delta["baseline"], delta["value"], delta["diff"], delta["tolerance"]))
+
+
def _cmd_validate(args) -> int:
dataset = evals.Dataset.from_jsonl(args.dataset)
questions = sorted({qid for example in dataset.examples for qid in example.questions})
@@ -139,9 +148,7 @@ def _cmd_run(args) -> int:
if args.baseline:
baseline = _load_report(args.baseline)
ok, deltas = report.compare(baseline, _parse_pairs(args.tolerance))
- for metric, delta in sorted(deltas.items()):
- print("%-18s baseline=%.4f value=%.4f diff=%+.4f (tol %.4f)"
- % (metric, delta["baseline"], delta["value"], delta["diff"], delta["tolerance"]))
+ _print_deltas(deltas)
if not ok:
failures.append("baseline comparison failed")
@@ -171,9 +178,7 @@ def _cmd_compare(args) -> int:
if k in ("config", "overall", "slices", "cases")})
baseline = _load_report(args.baseline)
ok, deltas = report.compare(baseline, _parse_pairs(args.tolerance))
- for metric, delta in sorted(deltas.items()):
- print("%-18s baseline=%.4f value=%.4f diff=%+.4f (tol %.4f)"
- % (metric, delta["baseline"], delta["value"], delta["diff"], delta["tolerance"]))
+ _print_deltas(deltas)
return 0 if ok else 1
diff --git a/research/evals/check_regression.py b/research/evals/check_regression.py
index 07457a8c..20167cb4 100644
--- a/research/evals/check_regression.py
+++ b/research/evals/check_regression.py
@@ -56,6 +56,10 @@ def main(argv: List[str] | None = None) -> int:
base = {"overall": _overall(baseline, language, args.baseline)}
passed, deltas = report.compare(base, tolerances)
for metric, delta in sorted(deltas.items()):
+ if delta.get("missing"):
+ print("%-6s %-16s baseline=%.4f missing from the report"
+ % (language, metric, delta["baseline"]))
+ continue
print("%-6s %-16s baseline=%.4f value=%.4f diff=%+.4f (tol %.4f)"
% (language, metric, delta["baseline"], delta["value"], delta["diff"], delta["tolerance"]))
if not passed:
diff --git a/tests/test_evals.py b/tests/test_evals.py
index 0e9ff2d7..4a8204ba 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -188,6 +188,30 @@ def test_compare_and_assert_regression():
assert_regression(report, baseline, {"ece": 0.01})
+def test_compare_fails_when_a_baseline_metric_is_missing():
+ # Every example errors under on_error="skip", so `overall` is empty. The gate used to skip
+ # each baseline metric absent from the report and pass with nothing compared.
+ class Broken:
+ def predict(self, state, questions, model=None):
+ raise RuntimeError("checkpoint failed to load")
+
+ report = evaluate(Broken(), Dataset([Example("s", Q, {"intent": "a"})] * 3), on_error="skip")
+ assert "choice_accuracy" not in report.overall
+ baseline = {"overall": {"choice_accuracy": 0.9, "ece": 0.05, "latency_p50_ms": 3.0}}
+ ok, deltas = report.compare(baseline, {"choice_accuracy": 1.0})
+ assert not ok
+ assert deltas["choice_accuracy"]["missing"] and deltas["ece"]["missing"]
+ assert "latency_p50_ms" not in deltas, "latency stays informational without a tolerance"
+ with pytest.raises(AssertionError, match="choice_accuracy"):
+ assert_regression(report, baseline)
+
+ # A partial loss fails too, and names only the metric that went missing.
+ partial = EvalReport(overall={"choice_accuracy": 0.9})
+ ok, deltas = partial.compare({"overall": {"choice_accuracy": 0.9, "noul_accuracy": 0.8}})
+ assert not ok and set(deltas) == {"choice_accuracy", "noul_accuracy"}
+ assert "missing" not in deltas["choice_accuracy"] and deltas["noul_accuracy"]["missing"]
+
+
def test_default_evaluators_cover_the_three_types():
names = {e.name for e in default_evaluators()}
assert {"choice_accuracy", "noul_accuracy", "score_mae", "mean_confidence"} <= names
@@ -224,6 +248,17 @@ def test_cli_compare_exit_codes(tmp_path):
"--tolerance", "choice_accuracy=0.001"]) == 1
+def test_cli_compare_fails_on_an_empty_report(tmp_path, capsys):
+ from laya import evals_cli
+
+ (tmp_path / "report.json").write_text(json.dumps({"overall": {}}))
+ (tmp_path / "baseline.json").write_text(json.dumps({"overall": {"choice_accuracy": 0.79}}))
+ assert evals_cli.main(["compare", str(tmp_path / "report.json"),
+ "--baseline", str(tmp_path / "baseline.json")]) == 1
+ out = capsys.readouterr().out
+ assert "choice_accuracy" in out and "missing from the report" in out
+
+
def test_cli_rejects_a_malformed_tolerance():
from laya import evals_cli