Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): use answer_confidence for calibration; ignore latency in compare

Emiliano Gandini Outedaabe9493a6da0
diff --git a/laya/evals.py b/laya/evals.py
index 2b26bf4b..cac581d6 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -139,8 +139,7 @@ class MeanConfidence(Evaluator):
name = "mean_confidence"
def score(self, answer, expected):
- confidence = answer.get("confidence")
- return float(confidence) if isinstance(confidence, (int, float)) else None
+ return _answer_confidence(answer)
DEFAULT_EVALUATORS = (ChoiceAccuracy, NoulAccuracy, ScoreMAE, MeanConfidence)
@@ -151,6 +150,15 @@ def default_evaluators() -> List[Evaluator]:
def _answer_confidence(answer: Dict[str, Any]) -> Optional[float]:
+ """The calibrated confidence Laya reports: `answer_confidence`, not the entropy score.
+
+ `answer["confidence"]` is entropy-based for choice and score, so calibration metrics must use
+ `answer_confidence`, which Laya reports on every answer type. The other keys are fallbacks for
+ a stripped-down result.
+ """
+ confidence = answer.get("answer_confidence")
+ if isinstance(confidence, (int, float)):
+ return float(confidence)
confidence = answer.get("confidence")
if isinstance(confidence, (int, float)):
return float(confidence)
@@ -226,6 +234,10 @@ class EvalReport:
for metric, base_value in base.items():
if metric not in self.overall:
continue
+ # Latency is informational; a re-run differs by timing noise, not quality, so it is
+ # compared only when a tolerance explicitly names it.
+ if metric.endswith("_ms") and metric not in tolerances:
+ continue
value = self.overall[metric]
diff = value - float(base_value)
allowed = float(tolerances.get(metric, 0.0))
diff --git a/tests/test_evals.py b/tests/test_evals.py
index 02bc4198..0e9ff2d7 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -64,6 +64,23 @@ def test_noul_and_score_math():
assert within.name == "score_within_0.5"
+def test_calibration_uses_answer_confidence():
+ answer = {"type": "choice", "choice": "a", "confidence": 0.2, "answer_confidence": 0.9}
+ assert MeanConfidence().score(answer, "a") == pytest.approx(0.9), "calibrated, not entropy"
+ report = evaluate(StubRunner({"s": {"intent": answer}}),
+ Dataset([Example("s", Q, {"intent": "a"})]))
+ assert report.overall["mean_confidence"] == pytest.approx(0.9)
+
+
+def test_compare_ignores_latency_by_default():
+ report = EvalReport(overall={"choice_accuracy": 0.8, "latency_p50_ms": 12.0})
+ baseline = {"overall": {"choice_accuracy": 0.8, "latency_p50_ms": 5.0}}
+ ok, deltas = report.compare(baseline)
+ assert ok and "latency_p50_ms" not in deltas, "timing noise is not a quality regression"
+ bad, deltas = report.compare(baseline, {"latency_p50_ms": 1.0})
+ assert not bad and "latency_p50_ms" in deltas
+
+
def test_ece_on_known_inputs():
assert ece([1.0, 1.0], [True, False]) == pytest.approx(0.5)
assert ece([0.0, 0.0], [False, False]) == pytest.approx(0.0)
@@ -188,7 +205,7 @@ def test_cli_validate_and_dispatch(tmp_path):
good = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])
assert evals_cli.main(["validate", good]) == 0
- assert cli.main(["eval", "validate", good]) == 0, "`laya eval` dispatches to laya-eval"
+ assert cli.main(["eval", "validate", good]) == 0, "`laya eval` dispatches to laya-evals"
bad = tmp_path / "bad.jsonl"
bad.write_text("{not json\n", encoding="utf-8")