Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): name the delta the gate failed on, including NaN and incomparable ones
PerryLink70293cd478a0
diff --git a/laya/evals.py b/laya/evals.pyindex acb4dab7..f0b1013a 100644--- a/laya/evals.py+++ b/laya/evals.py@@ -1389,7 +1389,14 @@ def assert_regression(report: EvalReport, baseline: Dict[str, Any],"""Raise AssertionError when `report` drifts from `baseline` beyond `tolerances`."""ok, deltas = report.compare(baseline, tolerances)if not ok:+ # `compare()` fails three shapes, and only the numeric one makes `abs(diff) > tolerance`+ # true: a missing metric and an incomparable coverage definition both store a NaN diff+ # (laya/evals.py:928-936), and every comparison with NaN is False, which is exactly why+ # compare() tests `not abs(diff) <= allowed` (comment at laya/evals.py:946-949). Filtering+ # on `>` alone dropped both, so the gate raised "evaluation regressed: {}" and lost the+ # metric and the reason compare() had already computed.failed = {m: d for m, d in deltas.items()- if d.get("missing") or abs(d["diff"]) > d["tolerance"]}+ if d.get("missing") or d.get("incomparable")+ or not abs(d["diff"]) <= d["tolerance"]}raise AssertionError("evaluation regressed: %s" % json.dumps(failed, sort_keys=True))return deltasdiff --git a/tests/test_evals.py b/tests/test_evals.pyindex f99bf615..5fe93193 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -3935,3 +3935,22 @@ def test_chunking_folds_criteria_key_order_on_the_request_form():evaluators=[ChoiceAccuracy()], batch_size=2)assert listed_report.config["timing"]["rows_grouped"] == 0, \"a reordered list criteria stays positional even for the request form"++++def test_assert_regression_names_incomparable_and_nan_deltas():+ """The gate must name every delta compare() failed, not just the numeric ones.++ An incomparable coverage definition and a NaN tolerance both fail compare() without+ `abs(diff) > tolerance` holding, so filtering the message on that comparison alone left+ `evaluation regressed: {}` with neither the metric nor the reason.+ """+ report = EvalReport(config={"coverage_metric_definition": 2}, overall={"aurc": 0.2})+ baseline = {"config": {"coverage_metric_definition": 1}, "overall": {"aurc": 0.1}}+ with pytest.raises(AssertionError, match="aurc"):+ assert_regression(report, baseline)++ with pytest.raises(AssertionError, match="choice_accuracy"):+ assert_regression(EvalReport(overall={"choice_accuracy": 0.8}),+ {"overall": {"choice_accuracy": 0.8}},+ {"choice_accuracy": float("nan")})