Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): use answer_confidence for calibration; ignore latency in compare
Emiliano Gandini Outedaabe9493a6da0
diff --git a/laya/evals.py b/laya/evals.pyindex 2b26bf4b..cac581d6 100644--- a/laya/evals.py+++ b/laya/evals.py@@ -139,8 +139,7 @@ class MeanConfidence(Evaluator):name = "mean_confidence"def score(self, answer, expected):- confidence = answer.get("confidence")- return float(confidence) if isinstance(confidence, (int, float)) else None+ return _answer_confidence(answer)DEFAULT_EVALUATORS = (ChoiceAccuracy, NoulAccuracy, ScoreMAE, MeanConfidence)@@ -151,6 +150,15 @@ def default_evaluators() -> List[Evaluator]:def _answer_confidence(answer: Dict[str, Any]) -> Optional[float]:+ """The calibrated confidence Laya reports: `answer_confidence`, not the entropy score.++ `answer["confidence"]` is entropy-based for choice and score, so calibration metrics must use+ `answer_confidence`, which Laya reports on every answer type. The other keys are fallbacks for+ a stripped-down result.+ """+ confidence = answer.get("answer_confidence")+ if isinstance(confidence, (int, float)):+ return float(confidence)confidence = answer.get("confidence")if isinstance(confidence, (int, float)):return float(confidence)@@ -226,6 +234,10 @@ class EvalReport:for metric, base_value in base.items():if metric not in self.overall:continue+ # Latency is informational; a re-run differs by timing noise, not quality, so it is+ # compared only when a tolerance explicitly names it.+ if metric.endswith("_ms") and metric not in tolerances:+ continuevalue = self.overall[metric]diff = value - float(base_value)allowed = float(tolerances.get(metric, 0.0))diff --git a/tests/test_evals.py b/tests/test_evals.pyindex 02bc4198..0e9ff2d7 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -64,6 +64,23 @@ def test_noul_and_score_math():assert within.name == "score_within_0.5"+def test_calibration_uses_answer_confidence():+ answer = {"type": "choice", "choice": "a", "confidence": 0.2, "answer_confidence": 0.9}+ assert MeanConfidence().score(answer, "a") == pytest.approx(0.9), "calibrated, not entropy"+ report = evaluate(StubRunner({"s": {"intent": answer}}),+ Dataset([Example("s", Q, {"intent": "a"})]))+ assert report.overall["mean_confidence"] == pytest.approx(0.9)+++def test_compare_ignores_latency_by_default():+ report = EvalReport(overall={"choice_accuracy": 0.8, "latency_p50_ms": 12.0})+ baseline = {"overall": {"choice_accuracy": 0.8, "latency_p50_ms": 5.0}}+ ok, deltas = report.compare(baseline)+ assert ok and "latency_p50_ms" not in deltas, "timing noise is not a quality regression"+ bad, deltas = report.compare(baseline, {"latency_p50_ms": 1.0})+ assert not bad and "latency_p50_ms" in deltas++def test_ece_on_known_inputs():assert ece([1.0, 1.0], [True, False]) == pytest.approx(0.5)assert ece([0.0, 0.0], [False, False]) == pytest.approx(0.0)@@ -188,7 +205,7 @@ def test_cli_validate_and_dispatch(tmp_path):good = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])assert evals_cli.main(["validate", good]) == 0- assert cli.main(["eval", "validate", good]) == 0, "`laya eval` dispatches to laya-eval"+ assert cli.main(["eval", "validate", good]) == 0, "`laya eval` dispatches to laya-evals"bad = tmp_path / "bad.jsonl"bad.write_text("{not json\n", encoding="utf-8")