Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): refuse a baseline the gate cannot subtract through
Aashish145e5568cf8b
diff --git a/laya/evals.py b/laya/evals.pyindex b6664e18..20f17a84 100644--- a/laya/evals.py+++ b/laya/evals.py@@ -66,6 +66,34 @@ class EvalError(ValueError):"""A dataset or report is malformed; the message names the row or field."""+def _metric_map(values: Any, label: str, *, allow_empty: bool = False) -> Dict[str, Any]:+ """Return a report's metric map, naming any shape `compare` cannot do arithmetic on.++ Both sides reach `compare` as JSON read from a file the caller named, and the loop subtracts+ one value from the other for every shared metric. A JSON scalar or list, an `overall` that is+ not an object, and a metric whose value is a string, null or dict each used to get that far:+ they raised `AttributeError`/`TypeError` as an unhandled traceback, and `laya-evals` exits 1+ for an unhandled traceback -- the code it reserves for a quality failure, so a broken baseline+ file printed the same verdict as a regressed model. A bool was worse than a crash: `float()`+ turns it into 1.0, so `"choice_accuracy": true` compared as a measurement of 100%.++ An empty map is refused unless `allow_empty`, because it has no metric left that could fail:+ the committed baseline that turns out to be `{}` or `null` would green the gate. `allow_empty`+ is the run's own side, where losing a metric a baseline still has is a result to report+ (`missing: True`) rather than a malformed document.+ """+ if not isinstance(values, dict):+ raise EvalError("%s metrics must be an object of metric name to number, got %s"+ % (label, type(values).__name__))+ if not values and not allow_empty:+ raise EvalError("%s has no metrics to compare, so nothing in it could fail" % label)+ for name, value in values.items():+ if isinstance(value, bool) or not isinstance(value, (int, float)):+ raise EvalError("%s metric %r is not a number, got %s"+ % (label, name, type(value).__name__))+ return values++def compare_agreement(reference: Dict, repeat: Dict, student: Dict, *,bootstrap_samples: int = 2000, seed: int = 0) -> Dict[str, Any]:"""Compare saved teacher reference, teacher repeat, and student choice observations.@@ -759,8 +787,19 @@ class EvalReport:every example errored under ``on_error="skip"`` has an empty ``overall``, and must notpass the gate by having nothing left to compare. A NaN metric, baseline or tolerancefails the comparison too.++ Either side that is not a metric name to number map, and a baseline with no metrics at+ all, raises ``EvalError`` rather than reaching the subtraction: both documents are read+ from a path the caller names, and a shape the gate cannot compare is a broken artifact.+ See ``_metric_map``."""- base = (baseline or {}).get("overall", baseline or {})+ # A baseline may be a full report document or the metric map on its own, which is the+ # shape `research/evals/check_regression.py` builds per language.+ if not isinstance(baseline, dict):+ raise EvalError("baseline report must be a JSON object, got %s"+ % type(baseline).__name__)+ base = _metric_map(baseline.get("overall", baseline), "baseline")+ overall = _metric_map(self.overall, "report", allow_empty=True)tolerances = tolerances or {}deltas: Dict[str, Dict[str, float]] = {}ok = True@@ -779,16 +818,16 @@ class EvalReport:if conflict is not None:deltas[metric] = {"baseline": float(base_value),- "value": float(self.overall.get(metric, float("nan"))),+ "value": float(overall.get(metric, float("nan"))),"diff": float("nan"), "tolerance": allowed, "incomparable": conflict}ok = Falsecontinue- if metric not in self.overall:+ if metric not in overall:deltas[metric] = {"baseline": float(base_value), "value": float("nan"),"diff": float("nan"), "tolerance": allowed, "missing": True}ok = Falsecontinue- value = self.overall[metric]+ value = overall[metric]diff = value - float(base_value)deltas[metric] = {"baseline": float(base_value), "value": value,"diff": diff, "tolerance": allowed}diff --git a/tests/test_evals.py b/tests/test_evals.pyindex cc469c2e..7a52e2c5 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -1687,6 +1687,85 @@ def test_a_nan_metric_fails_every_gate():assert healthy.compare({"overall": {"choice_accuracy": 0.9}})[0]+def test_compare_refuses_a_document_the_gate_cannot_subtract_through():+ """`--baseline` and the report path name files this function does arithmetic on.++ Every shape that is not a metric name to number map used to reach the subtraction and raise+ `AttributeError`/`TypeError` there, and a baseline that simply had no metrics compared nothing+ and returned a pass. A bool was the quietest of the group: `float(True)` is 1.0, so+ `"choice_accuracy": true` was read as a measurement of 100%.+ """+ report = EvalReport(overall={"choice_accuracy": 0.9})++ for baseline in (None, [], [{"accuracy": 0.9}], "reviewed baseline", 5, True):+ with pytest.raises(EvalError, match="baseline report must be a JSON object"):+ report.compare(baseline)++ for overall in (None, [], "abc", 5):+ with pytest.raises(EvalError, match="baseline metrics must be an object"):+ report.compare({"overall": overall})++ for empty in ({}, {"overall": {}}):+ with pytest.raises(EvalError, match="no metrics to compare"):+ report.compare(empty)++ # A report-shaped document with no `overall` falls back to its top level as the metric map, so+ # the schema tag is what gets subtracted.+ with pytest.raises(EvalError, match="baseline metric 'schema' is not a number"):+ report.compare({"schema": "laya-eval/1", "config": {}})+ for value, kind in (("0.9", "str"), (None, "NoneType"), ({"v": 1}, "dict"), (True, "bool")):+ with pytest.raises(EvalError, match="is not a number, got %s" % kind):+ report.compare({"overall": {"choice_accuracy": value}})++ # The candidate side is read from disk by the same command, so it is checked the same way.+ with pytest.raises(EvalError, match="report metric 'ece' is not a number"):+ EvalReport(overall={"ece": "0.02"}).compare({"overall": {"ece": 0.02}})++ # NaN stays a comparison, not a shape error: it fails the gate rather than refusing it.+ assert not report.compare({"overall": {"choice_accuracy": float("nan")}})[0]++ # And a run that lost a metric a baseline still has is still a result, not a usage error.+ ok, deltas = EvalReport(overall={}).compare({"overall": {"choice_accuracy": 0.9}})+ assert not ok and deltas["choice_accuracy"]["missing"]+++def test_cli_exits_2_for_a_malformed_baseline_or_report(tmp_path, capsys):+ """A broken artifact is exit code 2; exit 1 is what a quality failure is worth.++ Before, `laya-evals compare report.json --baseline {}` exited 0 with nothing compared, and+ `--baseline 5` died in a traceback that exited 1 -- indistinguishable from a regression to the+ CI job reading it.+ """+ from laya import evals_cli++ fresh = tmp_path / "report.json"+ fresh.write_text(json.dumps({"overall": {"choice_accuracy": 0.9}}))+ good = tmp_path / "good.json"+ good.write_text(json.dumps({"overall": {"choice_accuracy": 0.9}}))++ for raw in ("{}", "null", "[]", '"reviewed baseline"', "5", '{"overall": {}}',+ '{"overall": null}', '{"overall": {"choice_accuracy": "0.9"}}',+ '{"overall": {"choice_accuracy": true}}'):+ broken = tmp_path / "baseline.json"+ broken.write_text(raw)+ assert evals_cli.main(["compare", str(fresh), "--baseline", str(broken)]) == 2, raw+ # The candidate document carries the same shapes.+ for raw in ('{"overall": "abc"}', '{"overall": {"choice_accuracy": "0.9"}}'):+ broken = tmp_path / "candidate.json"+ broken.write_text(raw)+ assert evals_cli.main(["compare", str(broken), "--baseline", str(good)]) == 2, raw++ printed = capsys.readouterr().err+ assert "laya-evals: baseline has no metrics to compare" in printed+ assert "Traceback" not in printed++ # A well-formed pair still compares, and still exits 1 only when the numbers moved.+ assert evals_cli.main(["compare", str(fresh), "--baseline", str(good)]) == 0+ drifted = tmp_path / "drifted.json"+ drifted.write_text(json.dumps({"overall": {"choice_accuracy": 0.5}}))+ assert evals_cli.main(["compare", str(fresh), "--baseline", str(drifted)]) == 1++def test_cli_rejects_a_nan_limit_or_tolerance(capsys):from laya import evals_cli