Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(evals): selective-classification metrics (brier, aurc, selective accuracy)

AlKor13bf8d454cc4b4
diff --git a/docs/evals.md b/docs/evals.md
index 7a5ada2c..862caae8 100644
--- a/docs/evals.md
+++ b/docs/evals.md
@@ -132,6 +132,9 @@ Each metric is computed per answer where it applies and aggregated over the data
| `score_mae` | `score` | mean absolute error |
| `score_within_<tol>` | `score` | fraction within an absolute tolerance |
| `ece` | any answer with a confidence | expected calibration error, 15 bins, computed on `answer["answer_confidence"]`, the calibrated probability Laya reports on every answer type |
+| `brier` | any answer with a confidence and a known label | Brier score of confidence as P(correct), `mean((confidence - correct)**2)`; lower is better |
+| `aurc` | any answer with a confidence and a known label | area under the risk--coverage curve (mean selective risk over every coverage); lower is better, and rewards a confidence that *ranks* right from wrong rather than just being calibrated |
+| `selective_accuracy@50`, `selective_accuracy@80` | any answer with a confidence and a known label | accuracy over the most-confident 50% / 80% of answers -- what abstaining on the least-confident tail buys |
| `mean_confidence` | any answer with a confidence | mean reported `answer["answer_confidence"]` |
| `latency_p50_ms`, `latency_p95_ms` | per request | wall time each request waited, informational -- see [batching](#batching-and-timing) |
| `cost_per_decision_p50_ms`, `cost_per_decision_p95_ms` | per decision | a call's wall time divided by the rows it carried, informational |
diff --git a/laya/_eval_policy.py b/laya/_eval_policy.py
index ed8bb4d6..1742d1b3 100644
--- a/laya/_eval_policy.py
+++ b/laya/_eval_policy.py
@@ -80,7 +80,7 @@ def _count(report: evals.EvalReport, dimension: str, value: str, metric: str) ->
values = [case.get(dimension)]
if value not in [str(item) for item in values if item is not None]:
continue
- if metric == "ece":
+ if evals.is_confidence_metric(metric):
confidence, correct = case.get("confidence"), case.get("correct")
valid = _finite_number(confidence) and isinstance(correct, bool)
else:
diff --git a/laya/evals.py b/laya/evals.py
index 878a27f4..a9bc0b84 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -346,6 +346,64 @@ def ece(confidences: Sequence[float], corrects: Sequence[bool], bins: int = 15)
np.asarray(corrects, dtype=bool), bins=bins))
+# Selective-classification metrics. The abstention gate (#361/#456) and the per-bucket thresholds
+# (#394) decide *what to answer*; these say *how well the confidence ranks right from wrong* and
+# *what a coverage/risk trade buys*, which ECE (a calibration number) does not. All read the same
+# `(confidence, correct)` pairs ECE does, and are pure NumPy / torch-free.
+def brier(confidences: Sequence[float], corrects: Sequence[bool]) -> Optional[float]:
+ """Brier score of confidence as P(correct): mean((confidence - correct)**2). Lower is better."""
+ if not confidences:
+ return None
+ c = np.asarray(confidences, dtype=float)
+ y = np.asarray(corrects, dtype=float)
+ return float(np.mean((c - y) ** 2))
+
+
+def _risk_coverage(confidences: Sequence[float], corrects: Sequence[bool]):
+ """(coverage, risk) over the most-confident-first ordering; coverage k/n, risk = error@top-k."""
+ order = np.argsort(-np.asarray(confidences, dtype=float), kind="mergesort") # desc, stable
+ y = np.asarray(corrects, dtype=bool)[order].astype(float)
+ n = len(y)
+ k = np.arange(1, n + 1)
+ risk = 1.0 - np.cumsum(y) / k
+ return k / n, risk
+
+
+def aurc(confidences: Sequence[float], corrects: Sequence[bool]) -> Optional[float]:
+ """Area Under the Risk-Coverage curve (mean selective risk over every coverage). Lower is better.
+
+ A classifier whose confidence perfectly ranks right from wrong drives AURC toward the overall
+ error rate's area under the ideal curve; a confidence no better than random leaves it at the
+ base error rate. It rewards a confidence that *orders* answers, which calibration (ECE) does not.
+ """
+ if not confidences:
+ return None
+ _coverage, risk = _risk_coverage(confidences, corrects)
+ return float(np.mean(risk))
+
+
+def selective_accuracy(confidences: Sequence[float], corrects: Sequence[bool],
+ coverage: float) -> Optional[float]:
+ """Accuracy over the most-confident `coverage` fraction of answers (0 < coverage <= 1)."""
+ if not confidences or not 0.0 < coverage <= 1.0:
+ return None
+ import math as _math
+ order = np.argsort(-np.asarray(confidences, dtype=float), kind="mergesort")
+ y = np.asarray(corrects, dtype=bool)[order].astype(float)
+ k = max(1, int(_math.ceil(coverage * len(y))))
+ return float(np.mean(y[:k]))
+
+
+#: Coverage points reported as `selective_accuracy@NN`.
+SELECTIVE_COVERAGES = (0.5, 0.8)
+
+
+def is_confidence_metric(name: str) -> bool:
+ """True for a dataset-level metric computed from `(confidence, correct)` pairs, not per-answer
+ `scores` -- ECE and the selective-classification metrics. Policy counting treats these alike."""
+ return name in ("ece", "brier", "aurc") or name.startswith("selective_accuracy@")
+
+
def _answered_model(result: Any) -> Optional[Any]:
"""Which checkpoint produced `result`, read the way each runner records it.
@@ -490,9 +548,14 @@ def _aggregate(cases: Sequence[Dict[str, Any]], evaluators: Sequence[Evaluator])
paired = [(c["confidence"], c["correct"]) for c in cases
if c.get("confidence") is not None and c.get("correct") is not None]
if paired:
- value = ece([p[0] for p in paired], [p[1] for p in paired])
- if value is not None and not np.isnan(value):
- out["ece"] = value
+ confs = [p[0] for p in paired]
+ corrs = [p[1] for p in paired]
+ computed = {"ece": ece(confs, corrs), "brier": brier(confs, corrs), "aurc": aurc(confs, corrs)}
+ for cov in SELECTIVE_COVERAGES:
+ computed["selective_accuracy@%d" % round(cov * 100)] = selective_accuracy(confs, corrs, cov)
+ for name, value in computed.items():
+ if value is not None and not np.isnan(value):
+ out[name] = float(value)
return out
diff --git a/tests/test_evals.py b/tests/test_evals.py
index 249060ec..e624557d 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -20,9 +20,13 @@ from laya.evals import (
ScoreMAE,
ScoreWithin,
assert_regression,
+ aurc,
+ brier,
default_evaluators,
ece,
evaluate,
+ is_confidence_metric,
+ selective_accuracy,
)
Q = {"intent": {"type": "choice", "instructions": "?", "criteria": {"a": "x", "b": "y"}}}
@@ -94,6 +98,37 @@ def test_ece_on_known_inputs():
assert ece([], []) is None
+def test_selective_metrics_on_known_inputs():
+ # Brier of confidence-as-P(correct)
+ assert brier([1.0, 0.0], [True, False]) == pytest.approx(0.0)
+ assert brier([1.0, 1.0], [True, False]) == pytest.approx(0.5)
+ assert brier([], []) is None
+ # A confidence that ranks right-from-wrong perfectly: top-2 both correct, bottom-2 both wrong.
+ conf, corr = [0.9, 0.8, 0.2, 0.1], [True, True, False, False]
+ # risk@coverage = 1 - cum_correct/k over k=1..4 -> [0, 0, 1/3, 1/2]; AURC = mean = 0.2083
+ assert aurc(conf, corr) == pytest.approx((0 + 0 + 1 / 3 + 1 / 2) / 4, abs=1e-6)
+ assert selective_accuracy(conf, corr, 0.5) == pytest.approx(1.0) # top 2 are both correct
+ assert selective_accuracy(conf, corr, 1.0) == pytest.approx(0.5) # all four -> 2/4
+ assert selective_accuracy([], [], 0.5) is None
+ assert selective_accuracy(conf, corr, 0.0) is None # coverage must be in (0, 1]
+ # a worse ranking has higher AURC on the same accuracy
+ assert aurc([0.1, 0.2, 0.9, 0.8], corr) > aurc(conf, corr)
+ for name in ("ece", "brier", "aurc", "selective_accuracy@50"):
+ assert is_confidence_metric(name)
+ assert not is_confidence_metric("choice_accuracy")
+
+
+def test_selective_metrics_reach_the_report():
+ # choice answers with a spread of confidence and correctness so the pairs exist
+ ds = Dataset([Example("s1", Q, {"intent": "a"}), Example("s2", Q, {"intent": "a"}),
+ Example("s3", Q, {"intent": "a"}), Example("s4", Q, {"intent": "a"})])
+ answers = {"s1": {"intent": choice_answer("a", 0.95)}, "s2": {"intent": choice_answer("a", 0.80)},
+ "s3": {"intent": choice_answer("b", 0.60)}, "s4": {"intent": choice_answer("b", 0.30)}}
+ report = evaluate(StubRunner(answers), ds, evaluators=[ChoiceAccuracy()])
+ for name in ("ece", "brier", "aurc", "selective_accuracy@50", "selective_accuracy@80"):
+ assert name in report.overall, (name, sorted(report.overall))
+
+
def test_percentiles_use_nearest_rank():
from laya.evals import _percentiles
diff --git a/tests/test_evals_api.py b/tests/test_evals_api.py
index 6e73f2e2..3a031d4b 100644
--- a/tests/test_evals_api.py
+++ b/tests/test_evals_api.py
@@ -43,7 +43,8 @@ check_true("ScoreWithin names its tolerance", evals.ScoreWithin(0.25).name == "s
# --------------------------------------------------------------- exports / callables
for name in ("Dataset", "Example", "EvalError", "EvalReport", "evaluate", "ece", "assert_regression",
- "REPORT_SCHEMA", "questions_fingerprint", "file_fingerprint"):
+ "REPORT_SCHEMA", "questions_fingerprint", "file_fingerprint",
+ "brier", "aurc", "selective_accuracy", "is_confidence_metric"):
check_true("laya.evals.%s exists" % name, hasattr(evals, name))
check("REPORT_SCHEMA", evals.REPORT_SCHEMA, "laya-evals-report/1")