Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): refuse a baseline the gate cannot subtract through

Aashish145e5568cf8b
diff --git a/laya/evals.py b/laya/evals.py
index b6664e18..20f17a84 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -66,6 +66,34 @@ class EvalError(ValueError):
"""A dataset or report is malformed; the message names the row or field."""
+def _metric_map(values: Any, label: str, *, allow_empty: bool = False) -> Dict[str, Any]:
+ """Return a report's metric map, naming any shape `compare` cannot do arithmetic on.
+
+ Both sides reach `compare` as JSON read from a file the caller named, and the loop subtracts
+ one value from the other for every shared metric. A JSON scalar or list, an `overall` that is
+ not an object, and a metric whose value is a string, null or dict each used to get that far:
+ they raised `AttributeError`/`TypeError` as an unhandled traceback, and `laya-evals` exits 1
+ for an unhandled traceback -- the code it reserves for a quality failure, so a broken baseline
+ file printed the same verdict as a regressed model. A bool was worse than a crash: `float()`
+ turns it into 1.0, so `"choice_accuracy": true` compared as a measurement of 100%.
+
+ An empty map is refused unless `allow_empty`, because it has no metric left that could fail:
+ the committed baseline that turns out to be `{}` or `null` would green the gate. `allow_empty`
+ is the run's own side, where losing a metric a baseline still has is a result to report
+ (`missing: True`) rather than a malformed document.
+ """
+ if not isinstance(values, dict):
+ raise EvalError("%s metrics must be an object of metric name to number, got %s"
+ % (label, type(values).__name__))
+ if not values and not allow_empty:
+ raise EvalError("%s has no metrics to compare, so nothing in it could fail" % label)
+ for name, value in values.items():
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
+ raise EvalError("%s metric %r is not a number, got %s"
+ % (label, name, type(value).__name__))
+ return values
+
+
def compare_agreement(reference: Dict, repeat: Dict, student: Dict, *,
bootstrap_samples: int = 2000, seed: int = 0) -> Dict[str, Any]:
"""Compare saved teacher reference, teacher repeat, and student choice observations.
@@ -759,8 +787,19 @@ class EvalReport:
every example errored under ``on_error="skip"`` has an empty ``overall``, and must not
pass the gate by having nothing left to compare. A NaN metric, baseline or tolerance
fails the comparison too.
+
+ Either side that is not a metric name to number map, and a baseline with no metrics at
+ all, raises ``EvalError`` rather than reaching the subtraction: both documents are read
+ from a path the caller names, and a shape the gate cannot compare is a broken artifact.
+ See ``_metric_map``.
"""
- base = (baseline or {}).get("overall", baseline or {})
+ # A baseline may be a full report document or the metric map on its own, which is the
+ # shape `research/evals/check_regression.py` builds per language.
+ if not isinstance(baseline, dict):
+ raise EvalError("baseline report must be a JSON object, got %s"
+ % type(baseline).__name__)
+ base = _metric_map(baseline.get("overall", baseline), "baseline")
+ overall = _metric_map(self.overall, "report", allow_empty=True)
tolerances = tolerances or {}
deltas: Dict[str, Dict[str, float]] = {}
ok = True
@@ -779,16 +818,16 @@ class EvalReport:
if conflict is not None:
deltas[metric] = {
"baseline": float(base_value),
- "value": float(self.overall.get(metric, float("nan"))),
+ "value": float(overall.get(metric, float("nan"))),
"diff": float("nan"), "tolerance": allowed, "incomparable": conflict}
ok = False
continue
- if metric not in self.overall:
+ if metric not in overall:
deltas[metric] = {"baseline": float(base_value), "value": float("nan"),
"diff": float("nan"), "tolerance": allowed, "missing": True}
ok = False
continue
- value = self.overall[metric]
+ value = overall[metric]
diff = value - float(base_value)
deltas[metric] = {"baseline": float(base_value), "value": value,
"diff": diff, "tolerance": allowed}
diff --git a/tests/test_evals.py b/tests/test_evals.py
index cc469c2e..7a52e2c5 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -1687,6 +1687,85 @@ def test_a_nan_metric_fails_every_gate():
assert healthy.compare({"overall": {"choice_accuracy": 0.9}})[0]
+def test_compare_refuses_a_document_the_gate_cannot_subtract_through():
+ """`--baseline` and the report path name files this function does arithmetic on.
+
+ Every shape that is not a metric name to number map used to reach the subtraction and raise
+ `AttributeError`/`TypeError` there, and a baseline that simply had no metrics compared nothing
+ and returned a pass. A bool was the quietest of the group: `float(True)` is 1.0, so
+ `"choice_accuracy": true` was read as a measurement of 100%.
+ """
+ report = EvalReport(overall={"choice_accuracy": 0.9})
+
+ for baseline in (None, [], [{"accuracy": 0.9}], "reviewed baseline", 5, True):
+ with pytest.raises(EvalError, match="baseline report must be a JSON object"):
+ report.compare(baseline)
+
+ for overall in (None, [], "abc", 5):
+ with pytest.raises(EvalError, match="baseline metrics must be an object"):
+ report.compare({"overall": overall})
+
+ for empty in ({}, {"overall": {}}):
+ with pytest.raises(EvalError, match="no metrics to compare"):
+ report.compare(empty)
+
+ # A report-shaped document with no `overall` falls back to its top level as the metric map, so
+ # the schema tag is what gets subtracted.
+ with pytest.raises(EvalError, match="baseline metric 'schema' is not a number"):
+ report.compare({"schema": "laya-eval/1", "config": {}})
+ for value, kind in (("0.9", "str"), (None, "NoneType"), ({"v": 1}, "dict"), (True, "bool")):
+ with pytest.raises(EvalError, match="is not a number, got %s" % kind):
+ report.compare({"overall": {"choice_accuracy": value}})
+
+ # The candidate side is read from disk by the same command, so it is checked the same way.
+ with pytest.raises(EvalError, match="report metric 'ece' is not a number"):
+ EvalReport(overall={"ece": "0.02"}).compare({"overall": {"ece": 0.02}})
+
+ # NaN stays a comparison, not a shape error: it fails the gate rather than refusing it.
+ assert not report.compare({"overall": {"choice_accuracy": float("nan")}})[0]
+
+ # And a run that lost a metric a baseline still has is still a result, not a usage error.
+ ok, deltas = EvalReport(overall={}).compare({"overall": {"choice_accuracy": 0.9}})
+ assert not ok and deltas["choice_accuracy"]["missing"]
+
+
+def test_cli_exits_2_for_a_malformed_baseline_or_report(tmp_path, capsys):
+ """A broken artifact is exit code 2; exit 1 is what a quality failure is worth.
+
+ Before, `laya-evals compare report.json --baseline {}` exited 0 with nothing compared, and
+ `--baseline 5` died in a traceback that exited 1 -- indistinguishable from a regression to the
+ CI job reading it.
+ """
+ from laya import evals_cli
+
+ fresh = tmp_path / "report.json"
+ fresh.write_text(json.dumps({"overall": {"choice_accuracy": 0.9}}))
+ good = tmp_path / "good.json"
+ good.write_text(json.dumps({"overall": {"choice_accuracy": 0.9}}))
+
+ for raw in ("{}", "null", "[]", '"reviewed baseline"', "5", '{"overall": {}}',
+ '{"overall": null}', '{"overall": {"choice_accuracy": "0.9"}}',
+ '{"overall": {"choice_accuracy": true}}'):
+ broken = tmp_path / "baseline.json"
+ broken.write_text(raw)
+ assert evals_cli.main(["compare", str(fresh), "--baseline", str(broken)]) == 2, raw
+ # The candidate document carries the same shapes.
+ for raw in ('{"overall": "abc"}', '{"overall": {"choice_accuracy": "0.9"}}'):
+ broken = tmp_path / "candidate.json"
+ broken.write_text(raw)
+ assert evals_cli.main(["compare", str(broken), "--baseline", str(good)]) == 2, raw
+
+ printed = capsys.readouterr().err
+ assert "laya-evals: baseline has no metrics to compare" in printed
+ assert "Traceback" not in printed
+
+ # A well-formed pair still compares, and still exits 1 only when the numbers moved.
+ assert evals_cli.main(["compare", str(fresh), "--baseline", str(good)]) == 0
+ drifted = tmp_path / "drifted.json"
+ drifted.write_text(json.dumps({"overall": {"choice_accuracy": 0.5}}))
+ assert evals_cli.main(["compare", str(fresh), "--baseline", str(drifted)]) == 1
+
+
def test_cli_rejects_a_nan_limit_or_tolerance(capsys):
from laya import evals_cli