Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): refuse a tolerance that cannot fail the gate

Aashish0f50a07c844f
diff --git a/docs/evals.md b/docs/evals.md
index d9be6c21..6b3883a1 100644
--- a/docs/evals.md
+++ b/docs/evals.md
@@ -548,7 +548,9 @@ caller driving `laya.evals.evaluate` directly gets the same identity a CLI run d
- Keep the dataset, a baseline report (`--json` output you have reviewed), and the tolerances
together, committed, so a change is a reviewable diff. `--tolerance METRIC=VALUE` is the
- maximum absolute drift allowed for that metric.
+ maximum absolute drift allowed for that metric. It has to be a limit that can fail: `Infinity`
+ is one every finite drift fits inside, so it is refused rather than read as no gate for that
+ metric.
- `laya-evals run ... --baseline baseline.json --tolerance ...` exits non-zero on drift, so it
drops into CI unchanged. `laya.evals.EvalReport.compare` and `assert_regression` expose the
same logic for tests.
diff --git a/laya/evals.py b/laya/evals.py
index 20f17a84..d1c6fe79 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -94,6 +94,42 @@ def _metric_map(values: Any, label: str, *, allow_empty: bool = False) -> Dict[s
return values
+def _tolerance_map(tolerances: Any) -> Dict[str, Any]:
+ """Return a caller's tolerance map, naming any entry `compare` cannot gate on.
+
+ Both metric maps go through ``_metric_map`` for this reason; the tolerances reach the same
+ subtraction as a dict the caller built -- a hand-edited `thresholds.json` for
+ `research/evals/check_regression.py`, a config module for a test -- and were read with a bare
+ ``float(tolerances.get(metric, 0.0))``. A string, null, list or dict there raised ``TypeError``
+ from inside the arithmetic and a list or string given as the whole map raised
+ ``AttributeError``, so the gate died with exit 1 -- the code `laya-evals` reserves for a
+ quality failure -- on what is a broken gate parameter. A bool was read as 1.0, which for a
+ metric that lives in [0, 1] accepts any drift the run can report.
+
+ ``Infinity`` is the one number that cannot fail a comparison: every finite drift is inside it,
+ so naming a metric with it turns that metric's gate into a no-op while the run still exits 0
+ -- the code that means "the gate ran and the numbers passed". A NaN or a negative tolerance
+ decides the opposite way (every comparison with NaN is False, and no absolute difference is
+ ``<=`` a negative number), so those stay results rather than refusals; see
+ ``test_compare_refuses_a_tolerance_that_cannot_fail``. An empty map is a complete answer here,
+ unlike an empty metric map: it means every shared metric must match exactly.
+ """
+ if tolerances is None:
+ return {}
+ if not isinstance(tolerances, dict):
+ raise EvalError("tolerances must be an object of metric name to number, got %s"
+ % type(tolerances).__name__)
+ import math as _math
+ for name, value in tolerances.items():
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
+ raise EvalError("tolerance for %r is not a number, got %s"
+ % (name, type(value).__name__))
+ if _math.isinf(value) and value > 0:
+ raise EvalError("tolerance for %r is Infinity, so it accepts a drift of any size"
+ % name)
+ return tolerances
+
+
def compare_agreement(reference: Dict, repeat: Dict, student: Dict, *,
bootstrap_samples: int = 2000, seed: int = 0) -> Dict[str, Any]:
"""Compare saved teacher reference, teacher repeat, and student choice observations.
@@ -788,10 +824,11 @@ class EvalReport:
pass the gate by having nothing left to compare. A NaN metric, baseline or tolerance
fails the comparison too.
- Either side that is not a metric name to number map, and a baseline with no metrics at
- all, raises ``EvalError`` rather than reaching the subtraction: both documents are read
- from a path the caller names, and a shape the gate cannot compare is a broken artifact.
- See ``_metric_map``.
+ Either side that is not a metric name to number map, a baseline with no metrics at all,
+ and a tolerance that is not a number or is ``Infinity`` raise ``EvalError`` rather than
+ reaching the subtraction: both documents are read from a path the caller names, and a shape
+ the gate cannot compare -- or a limit that cannot fail -- is a broken artifact.
+ See ``_metric_map`` and ``_tolerance_map``.
"""
# A baseline may be a full report document or the metric map on its own, which is the
# shape `research/evals/check_regression.py` builds per language.
@@ -800,7 +837,7 @@ class EvalReport:
% type(baseline).__name__)
base = _metric_map(baseline.get("overall", baseline), "baseline")
overall = _metric_map(self.overall, "report", allow_empty=True)
- tolerances = tolerances or {}
+ tolerances = _tolerance_map(tolerances)
deltas: Dict[str, Dict[str, float]] = {}
ok = True
for metric, base_value in base.items():
diff --git a/tests/test_evals.py b/tests/test_evals.py
index f8ad6564..2c06d6cd 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -1729,6 +1729,68 @@ def test_compare_refuses_a_document_the_gate_cannot_subtract_through():
assert not ok and deltas["choice_accuracy"]["missing"]
+def test_compare_refuses_a_tolerance_that_cannot_fail():
+ """The caller's tolerance map is the third document the gate does arithmetic on.
+
+ The metric maps have refused broken shapes since ``_metric_map``; the tolerances were read with
+ a bare ``float(tolerances.get(metric, 0.0))`` inside the loop. They arrive as a dict the caller
+ built -- `research/evals/check_regression.py` reads them from a committed thresholds.json with
+ ``json.load``, which parses an ``Infinity`` literal -- so this is the same artifact class, on the
+ side that decides whether the gate can fail at all.
+ """
+ report = EvalReport(overall={"choice_accuracy": 0.8})
+ baseline = {"overall": {"choice_accuracy": 0.90}}
+
+ # The harm: a limit every possible drift fits inside greens a 0.10 drop, and the run still
+ # exits 0 with the line that says the gate ran. `1e999` is the spelling that arrives without
+ # anyone typing "inf" -- it overflows to Infinity and raises nothing.
+ for value in (float("inf"), 1e999):
+ with pytest.raises(EvalError, match="accepts a drift of any size") as exc:
+ report.compare(baseline, {"choice_accuracy": value})
+ assert "'choice_accuracy'" in str(exc.value), "the message names the metric it refuses"
+
+ # A bool was the quietest of the rest: `float(True)` is 1.0, which is a tolerance no
+ # [0, 1] metric can breach. Everything else used to die in the subtraction as a TypeError.
+ for value, kind in ((True, "bool"), (False, "bool"), ("0.1", "str"), (None, "NoneType"),
+ ([0.1], "list"), ({"v": 1}, "dict"), (0j, "complex")):
+ with pytest.raises(EvalError, match="is not a number, got %s" % kind):
+ report.compare(baseline, {"choice_accuracy": value})
+
+ # The map itself: a list or string of pairs, a number, and the empty tuple all used to reach
+ # `.get`, and the falsy ones (`0`, `()`) were read as "no tolerances" by `tolerances or {}`.
+ for bad in ([("choice_accuracy", 0.1)], "0.1", 5, 0, ()):
+ with pytest.raises(EvalError, match="tolerances must be an object"):
+ report.compare(baseline, bad)
+
+ # Checked for every entry, not only the ones this baseline happens to consult: a gate
+ # parameter that is broken is broken whether or not the run reaches it.
+ with pytest.raises(EvalError, match="accepts a drift of any size"):
+ report.compare(baseline, {"ece_ms": float("inf")})
+ # And not only the first entry: this is a committed thresholds.json where one metric's limit
+ # was replaced by a value that cannot fail, next to a limit the run does check and passes.
+ both = EvalReport(overall={"choice_accuracy": 0.8, "ece": 0.10})
+ with pytest.raises(EvalError, match="accepts a drift of any size"):
+ both.compare({"overall": {"choice_accuracy": 0.90, "ece": 0.08}},
+ {"choice_accuracy": 0.2, "ece": float("inf")})
+
+ # `assert_regression` is the other public door, and it refuses rather than raising the
+ # AssertionError its callers catch on a real regression.
+ with pytest.raises(EvalError, match="accepts a drift of any size"):
+ assert_regression(report, baseline, {"choice_accuracy": float("inf")})
+
+ # The boundary that stays a comparison: NaN, -Infinity and a negative number each fail every
+ # `abs(diff) <= allowed` test, so they report a gate that did not pass rather than refuse one.
+ assert not report.compare(baseline, {"choice_accuracy": float("nan")})[0]
+ assert not report.compare(baseline, {"choice_accuracy": float("-inf")})[0]
+ assert not report.compare(baseline, {"choice_accuracy": -0.5})[0]
+
+ # And no tolerances at all is a complete answer: everything shared must match exactly.
+ assert not report.compare(baseline)[0]
+ assert not report.compare(baseline, {})[0]
+ assert not report.compare(baseline, None)[0]
+ assert report.compare({"overall": {"choice_accuracy": 0.8}}, {"choice_accuracy": 0.2})[0]
+
+
def test_cli_exits_2_for_a_malformed_baseline_or_report(tmp_path, capsys):
"""A broken artifact is exit code 2; exit 1 is what a quality failure is worth.
@@ -1818,20 +1880,19 @@ def test_cli_rejects_an_infinite_limit_or_tolerance(capsys):
def test_an_infinite_limit_would_have_decided_nothing():
"""The harm the guard above prevents, pinned so relaxing it cannot go quiet.
- Without the refusal these are the values `_check_thresholds` and `compare` compute from a
- non-finite limit, and each one is a pass: `laya-evals` would exit 0 having gated nothing. A
- test that asserts them is not asserting correct behaviour -- it is the witness that the
- guard's absence was a defect.
+ Without the refusal these are the values `_check_thresholds` computes from a non-finite limit,
+ and each one is a pass: `laya-evals` would exit 0 having gated nothing. A test that asserts them
+ is not asserting correct behaviour -- it is the witness that the guard's absence was a defect.
+
+ `compare` had the same hole on its tolerance map, one layer below the CLI guard, and it no
+ longer computes a pass from Infinity: it refuses it. See
+ `test_compare_refuses_a_tolerance_that_cannot_fail`.
"""
from laya import evals_cli
inf = float("inf")
assert evals_cli._check_thresholds({"ece": 0.42}, {}, {"ece": inf}) == []
assert evals_cli._check_thresholds({"choice_accuracy": 0.10}, {"choice_accuracy": -inf}, {}) == []
- ok, deltas = EvalReport(config={}, overall={"choice_accuracy": 0.10}).compare(
- {"overall": {"choice_accuracy": 0.90}}, {"choice_accuracy": inf})
- assert ok, "an 0.80 drop passed a gate whose tolerance was inf"
- assert deltas["choice_accuracy"]["diff"] == pytest.approx(-0.8)
def test_default_evaluators_cover_the_three_types():