Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(evals): baseline regression check for laya_eval reports

Emiliano Gandini Outedabda44a1edeab
diff --git a/research/evals/check_regression.py b/research/evals/check_regression.py
new file mode 100644
index 00000000..07457a8c
--- /dev/null
+++ b/research/evals/check_regression.py
@@ -0,0 +1,68 @@
+"""Compare a fresh `laya_eval` report to a committed baseline and fail on drift.
+
+Used by the scheduled `evals` workflow: it runs `research/eval/laya_eval.py`, then points this
+at the result and a baseline in `research/results/`. The comparison uses
+`laya.evals.EvalReport.compare`, so the tolerance semantics match the CLI.
+
+ python research/evals/check_regression.py fresh.json baseline.json thresholds.json
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from typing import Any, Dict, List
+
+_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+if _REPO_ROOT not in sys.path:
+ sys.path.insert(0, _REPO_ROOT)
+
+from laya.evals import EvalReport # noqa: E402
+
+METRICS = ("accuracy", "ece", "mean_confidence")
+
+
+def _overall(document: Dict[str, Any], language: str, where: str) -> Dict[str, float]:
+ report = (document or {}).get("report") or {}
+ if language not in report:
+ raise SystemExit("%s has no report for language %r" % (where, language))
+ entry = report[language]
+ return {metric: float(entry[metric]) for metric in METRICS if entry.get(metric) is not None}
+
+
+def main(argv: List[str] | None = None) -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("fresh", help="the report produced by research/eval/laya_eval.py")
+ parser.add_argument("baseline", help="a committed baseline report")
+ parser.add_argument("thresholds", nargs="?", help="thresholds.json (languages + tolerance)")
+ parser.add_argument("--lang", action="append", help="language to check; repeatable")
+ args = parser.parse_args(argv)
+
+ config: Dict[str, Any] = {}
+ if args.thresholds:
+ with open(args.thresholds, "r", encoding="utf-8") as handle:
+ config = json.load(handle)
+ languages = args.lang or config.get("languages") or ["en"]
+ tolerances = config.get("tolerance") or {}
+ with open(args.fresh, "r", encoding="utf-8") as handle:
+ fresh = json.load(handle)
+ with open(args.baseline, "r", encoding="utf-8") as handle:
+ baseline = json.load(handle)
+
+ ok = True
+ for language in languages:
+ report = EvalReport(overall=_overall(fresh, language, args.fresh))
+ base = {"overall": _overall(baseline, language, args.baseline)}
+ passed, deltas = report.compare(base, tolerances)
+ for metric, delta in sorted(deltas.items()):
+ print("%-6s %-16s baseline=%.4f value=%.4f diff=%+.4f (tol %.4f)"
+ % (language, metric, delta["baseline"], delta["value"], delta["diff"], delta["tolerance"]))
+ if not passed:
+ ok = False
+ print("%-6s REGRESSED" % language, file=sys.stderr)
+ return 0 if ok else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())