Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): fail the baseline comparison when a metric is missing
can-gaa-hou605bebb2a746
diff --git a/laya/evals.py b/laya/evals.pyindex cac581d6..7c3e91bb 100644--- a/laya/evals.py+++ b/laya/evals.py@@ -225,22 +225,28 @@ class EvalReport:"""Compare `overall` to a baseline report's `overall`. Returns (ok, deltas).With no `tolerances`, every shared metric must match exactly; a tolerance is the maximum- absolute difference allowed for that metric.+ absolute difference allowed for that metric. A baseline metric the report no longer has+ fails the comparison (its delta carries ``missing: True`` and a NaN value): a run whose+ every example errored under ``on_error="skip"`` has an empty ``overall``, and must not+ pass the gate by having nothing left to compare."""base = (baseline or {}).get("overall", baseline or {})tolerances = tolerances or {}deltas: Dict[str, Dict[str, float]] = {}ok = Truefor metric, base_value in base.items():- if metric not in self.overall:- continue# Latency is informational; a re-run differs by timing noise, not quality, so it is# compared only when a tolerance explicitly names it.if metric.endswith("_ms") and metric not in tolerances:continue+ allowed = float(tolerances.get(metric, 0.0))+ if metric not in self.overall:+ deltas[metric] = {"baseline": float(base_value), "value": float("nan"),+ "diff": float("nan"), "tolerance": allowed, "missing": True}+ ok = False+ continuevalue = self.overall[metric]diff = value - float(base_value)- allowed = float(tolerances.get(metric, 0.0))deltas[metric] = {"baseline": float(base_value), "value": value,"diff": diff, "tolerance": allowed}if abs(diff) > allowed:@@ -366,6 +372,7 @@ def assert_regression(report: EvalReport, baseline: Dict[str, Any],"""Raise AssertionError when `report` drifts from `baseline` beyond `tolerances`."""ok, deltas = report.compare(baseline, tolerances)if not ok:- failed = {m: d for m, d in deltas.items() if abs(d["diff"]) > d["tolerance"]}+ failed = {m: d for m, d in deltas.items()+ if d.get("missing") or abs(d["diff"]) > d["tolerance"]}raise AssertionError("evaluation regressed: %s" % json.dumps(failed, sort_keys=True))return deltasdiff --git a/laya/evals_cli.py b/laya/evals_cli.pyindex e656eba6..1dbd44a6 100644--- a/laya/evals_cli.py+++ b/laya/evals_cli.py@@ -108,6 +108,15 @@ def _check_thresholds(overall: Dict[str, float], mins: Dict[str, float],return failures+def _print_deltas(deltas: Dict[str, Dict[str, Any]]) -> None:+ for metric, delta in sorted(deltas.items()):+ if delta.get("missing"):+ print("%-18s baseline=%.4f missing from the report" % (metric, delta["baseline"]))+ continue+ print("%-18s baseline=%.4f value=%.4f diff=%+.4f (tol %.4f)"+ % (metric, delta["baseline"], delta["value"], delta["diff"], delta["tolerance"]))++def _cmd_validate(args) -> int:dataset = evals.Dataset.from_jsonl(args.dataset)questions = sorted({qid for example in dataset.examples for qid in example.questions})@@ -139,9 +148,7 @@ def _cmd_run(args) -> int:if args.baseline:baseline = _load_report(args.baseline)ok, deltas = report.compare(baseline, _parse_pairs(args.tolerance))- for metric, delta in sorted(deltas.items()):- print("%-18s baseline=%.4f value=%.4f diff=%+.4f (tol %.4f)"- % (metric, delta["baseline"], delta["value"], delta["diff"], delta["tolerance"]))+ _print_deltas(deltas)if not ok:failures.append("baseline comparison failed")@@ -171,9 +178,7 @@ def _cmd_compare(args) -> int:if k in ("config", "overall", "slices", "cases")})baseline = _load_report(args.baseline)ok, deltas = report.compare(baseline, _parse_pairs(args.tolerance))- for metric, delta in sorted(deltas.items()):- print("%-18s baseline=%.4f value=%.4f diff=%+.4f (tol %.4f)"- % (metric, delta["baseline"], delta["value"], delta["diff"], delta["tolerance"]))+ _print_deltas(deltas)return 0 if ok else 1diff --git a/research/evals/check_regression.py b/research/evals/check_regression.pyindex 07457a8c..20167cb4 100644--- a/research/evals/check_regression.py+++ b/research/evals/check_regression.py@@ -56,6 +56,10 @@ def main(argv: List[str] | None = None) -> int:base = {"overall": _overall(baseline, language, args.baseline)}passed, deltas = report.compare(base, tolerances)for metric, delta in sorted(deltas.items()):+ if delta.get("missing"):+ print("%-6s %-16s baseline=%.4f missing from the report"+ % (language, metric, delta["baseline"]))+ continueprint("%-6s %-16s baseline=%.4f value=%.4f diff=%+.4f (tol %.4f)"% (language, metric, delta["baseline"], delta["value"], delta["diff"], delta["tolerance"]))if not passed:diff --git a/tests/test_evals.py b/tests/test_evals.pyindex 0e9ff2d7..4a8204ba 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -188,6 +188,30 @@ def test_compare_and_assert_regression():assert_regression(report, baseline, {"ece": 0.01})+def test_compare_fails_when_a_baseline_metric_is_missing():+ # Every example errors under on_error="skip", so `overall` is empty. The gate used to skip+ # each baseline metric absent from the report and pass with nothing compared.+ class Broken:+ def predict(self, state, questions, model=None):+ raise RuntimeError("checkpoint failed to load")++ report = evaluate(Broken(), Dataset([Example("s", Q, {"intent": "a"})] * 3), on_error="skip")+ assert "choice_accuracy" not in report.overall+ baseline = {"overall": {"choice_accuracy": 0.9, "ece": 0.05, "latency_p50_ms": 3.0}}+ ok, deltas = report.compare(baseline, {"choice_accuracy": 1.0})+ assert not ok+ assert deltas["choice_accuracy"]["missing"] and deltas["ece"]["missing"]+ assert "latency_p50_ms" not in deltas, "latency stays informational without a tolerance"+ with pytest.raises(AssertionError, match="choice_accuracy"):+ assert_regression(report, baseline)++ # A partial loss fails too, and names only the metric that went missing.+ partial = EvalReport(overall={"choice_accuracy": 0.9})+ ok, deltas = partial.compare({"overall": {"choice_accuracy": 0.9, "noul_accuracy": 0.8}})+ assert not ok and set(deltas) == {"choice_accuracy", "noul_accuracy"}+ assert "missing" not in deltas["choice_accuracy"] and deltas["noul_accuracy"]["missing"]++def test_default_evaluators_cover_the_three_types():names = {e.name for e in default_evaluators()}assert {"choice_accuracy", "noul_accuracy", "score_mae", "mean_confidence"} <= names@@ -224,6 +248,17 @@ def test_cli_compare_exit_codes(tmp_path):"--tolerance", "choice_accuracy=0.001"]) == 1+def test_cli_compare_fails_on_an_empty_report(tmp_path, capsys):+ from laya import evals_cli++ (tmp_path / "report.json").write_text(json.dumps({"overall": {}}))+ (tmp_path / "baseline.json").write_text(json.dumps({"overall": {"choice_accuracy": 0.79}}))+ assert evals_cli.main(["compare", str(tmp_path / "report.json"),+ "--baseline", str(tmp_path / "baseline.json")]) == 1+ out = capsys.readouterr().out+ assert "choice_accuracy" in out and "missing from the report" in out++def test_cli_rejects_a_malformed_tolerance():from laya import evals_cli