Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(evals): selective-classification metrics (brier, aurc, selective accuracy)
AlKor13bf8d454cc4b4
diff --git a/docs/evals.md b/docs/evals.mdindex 7a5ada2c..862caae8 100644--- a/docs/evals.md+++ b/docs/evals.md@@ -132,6 +132,9 @@ Each metric is computed per answer where it applies and aggregated over the data| `score_mae` | `score` | mean absolute error || `score_within_<tol>` | `score` | fraction within an absolute tolerance || `ece` | any answer with a confidence | expected calibration error, 15 bins, computed on `answer["answer_confidence"]`, the calibrated probability Laya reports on every answer type |+| `brier` | any answer with a confidence and a known label | Brier score of confidence as P(correct), `mean((confidence - correct)**2)`; lower is better |+| `aurc` | any answer with a confidence and a known label | area under the risk--coverage curve (mean selective risk over every coverage); lower is better, and rewards a confidence that *ranks* right from wrong rather than just being calibrated |+| `selective_accuracy@50`, `selective_accuracy@80` | any answer with a confidence and a known label | accuracy over the most-confident 50% / 80% of answers -- what abstaining on the least-confident tail buys || `mean_confidence` | any answer with a confidence | mean reported `answer["answer_confidence"]` || `latency_p50_ms`, `latency_p95_ms` | per request | wall time each request waited, informational -- see [batching](#batching-and-timing) || `cost_per_decision_p50_ms`, `cost_per_decision_p95_ms` | per decision | a call's wall time divided by the rows it carried, informational |diff --git a/laya/_eval_policy.py b/laya/_eval_policy.pyindex ed8bb4d6..1742d1b3 100644--- a/laya/_eval_policy.py+++ b/laya/_eval_policy.py@@ -80,7 +80,7 @@ def _count(report: evals.EvalReport, dimension: str, value: str, metric: str) ->values = [case.get(dimension)]if value not in [str(item) for item in values if item is not None]:continue- if metric == "ece":+ if evals.is_confidence_metric(metric):confidence, correct = case.get("confidence"), case.get("correct")valid = _finite_number(confidence) and isinstance(correct, bool)else:diff --git a/laya/evals.py b/laya/evals.pyindex 878a27f4..a9bc0b84 100644--- a/laya/evals.py+++ b/laya/evals.py@@ -346,6 +346,64 @@ def ece(confidences: Sequence[float], corrects: Sequence[bool], bins: int = 15)np.asarray(corrects, dtype=bool), bins=bins))+# Selective-classification metrics. The abstention gate (#361/#456) and the per-bucket thresholds+# (#394) decide *what to answer*; these say *how well the confidence ranks right from wrong* and+# *what a coverage/risk trade buys*, which ECE (a calibration number) does not. All read the same+# `(confidence, correct)` pairs ECE does, and are pure NumPy / torch-free.+def brier(confidences: Sequence[float], corrects: Sequence[bool]) -> Optional[float]:+ """Brier score of confidence as P(correct): mean((confidence - correct)**2). Lower is better."""+ if not confidences:+ return None+ c = np.asarray(confidences, dtype=float)+ y = np.asarray(corrects, dtype=float)+ return float(np.mean((c - y) ** 2))+++def _risk_coverage(confidences: Sequence[float], corrects: Sequence[bool]):+ """(coverage, risk) over the most-confident-first ordering; coverage k/n, risk = error@top-k."""+ order = np.argsort(-np.asarray(confidences, dtype=float), kind="mergesort") # desc, stable+ y = np.asarray(corrects, dtype=bool)[order].astype(float)+ n = len(y)+ k = np.arange(1, n + 1)+ risk = 1.0 - np.cumsum(y) / k+ return k / n, risk+++def aurc(confidences: Sequence[float], corrects: Sequence[bool]) -> Optional[float]:+ """Area Under the Risk-Coverage curve (mean selective risk over every coverage). Lower is better.++ A classifier whose confidence perfectly ranks right from wrong drives AURC toward the overall+ error rate's area under the ideal curve; a confidence no better than random leaves it at the+ base error rate. It rewards a confidence that *orders* answers, which calibration (ECE) does not.+ """+ if not confidences:+ return None+ _coverage, risk = _risk_coverage(confidences, corrects)+ return float(np.mean(risk))+++def selective_accuracy(confidences: Sequence[float], corrects: Sequence[bool],+ coverage: float) -> Optional[float]:+ """Accuracy over the most-confident `coverage` fraction of answers (0 < coverage <= 1)."""+ if not confidences or not 0.0 < coverage <= 1.0:+ return None+ import math as _math+ order = np.argsort(-np.asarray(confidences, dtype=float), kind="mergesort")+ y = np.asarray(corrects, dtype=bool)[order].astype(float)+ k = max(1, int(_math.ceil(coverage * len(y))))+ return float(np.mean(y[:k]))+++#: Coverage points reported as `selective_accuracy@NN`.+SELECTIVE_COVERAGES = (0.5, 0.8)+++def is_confidence_metric(name: str) -> bool:+ """True for a dataset-level metric computed from `(confidence, correct)` pairs, not per-answer+ `scores` -- ECE and the selective-classification metrics. Policy counting treats these alike."""+ return name in ("ece", "brier", "aurc") or name.startswith("selective_accuracy@")++def _answered_model(result: Any) -> Optional[Any]:"""Which checkpoint produced `result`, read the way each runner records it.@@ -490,9 +548,14 @@ def _aggregate(cases: Sequence[Dict[str, Any]], evaluators: Sequence[Evaluator])paired = [(c["confidence"], c["correct"]) for c in casesif c.get("confidence") is not None and c.get("correct") is not None]if paired:- value = ece([p[0] for p in paired], [p[1] for p in paired])- if value is not None and not np.isnan(value):- out["ece"] = value+ confs = [p[0] for p in paired]+ corrs = [p[1] for p in paired]+ computed = {"ece": ece(confs, corrs), "brier": brier(confs, corrs), "aurc": aurc(confs, corrs)}+ for cov in SELECTIVE_COVERAGES:+ computed["selective_accuracy@%d" % round(cov * 100)] = selective_accuracy(confs, corrs, cov)+ for name, value in computed.items():+ if value is not None and not np.isnan(value):+ out[name] = float(value)return outdiff --git a/tests/test_evals.py b/tests/test_evals.pyindex 249060ec..e624557d 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -20,9 +20,13 @@ from laya.evals import (ScoreMAE,ScoreWithin,assert_regression,+ aurc,+ brier,default_evaluators,ece,evaluate,+ is_confidence_metric,+ selective_accuracy,)Q = {"intent": {"type": "choice", "instructions": "?", "criteria": {"a": "x", "b": "y"}}}@@ -94,6 +98,37 @@ def test_ece_on_known_inputs():assert ece([], []) is None+def test_selective_metrics_on_known_inputs():+ # Brier of confidence-as-P(correct)+ assert brier([1.0, 0.0], [True, False]) == pytest.approx(0.0)+ assert brier([1.0, 1.0], [True, False]) == pytest.approx(0.5)+ assert brier([], []) is None+ # A confidence that ranks right-from-wrong perfectly: top-2 both correct, bottom-2 both wrong.+ conf, corr = [0.9, 0.8, 0.2, 0.1], [True, True, False, False]+ # risk@coverage = 1 - cum_correct/k over k=1..4 -> [0, 0, 1/3, 1/2]; AURC = mean = 0.2083+ assert aurc(conf, corr) == pytest.approx((0 + 0 + 1 / 3 + 1 / 2) / 4, abs=1e-6)+ assert selective_accuracy(conf, corr, 0.5) == pytest.approx(1.0) # top 2 are both correct+ assert selective_accuracy(conf, corr, 1.0) == pytest.approx(0.5) # all four -> 2/4+ assert selective_accuracy([], [], 0.5) is None+ assert selective_accuracy(conf, corr, 0.0) is None # coverage must be in (0, 1]+ # a worse ranking has higher AURC on the same accuracy+ assert aurc([0.1, 0.2, 0.9, 0.8], corr) > aurc(conf, corr)+ for name in ("ece", "brier", "aurc", "selective_accuracy@50"):+ assert is_confidence_metric(name)+ assert not is_confidence_metric("choice_accuracy")+++def test_selective_metrics_reach_the_report():+ # choice answers with a spread of confidence and correctness so the pairs exist+ ds = Dataset([Example("s1", Q, {"intent": "a"}), Example("s2", Q, {"intent": "a"}),+ Example("s3", Q, {"intent": "a"}), Example("s4", Q, {"intent": "a"})])+ answers = {"s1": {"intent": choice_answer("a", 0.95)}, "s2": {"intent": choice_answer("a", 0.80)},+ "s3": {"intent": choice_answer("b", 0.60)}, "s4": {"intent": choice_answer("b", 0.30)}}+ report = evaluate(StubRunner(answers), ds, evaluators=[ChoiceAccuracy()])+ for name in ("ece", "brier", "aurc", "selective_accuracy@50", "selective_accuracy@80"):+ assert name in report.overall, (name, sorted(report.overall))++def test_percentiles_use_nearest_rank():from laya.evals import _percentilesdiff --git a/tests/test_evals_api.py b/tests/test_evals_api.pyindex 6e73f2e2..3a031d4b 100644--- a/tests/test_evals_api.py+++ b/tests/test_evals_api.py@@ -43,7 +43,8 @@ check_true("ScoreWithin names its tolerance", evals.ScoreWithin(0.25).name == "s# --------------------------------------------------------------- exports / callablesfor name in ("Dataset", "Example", "EvalError", "EvalReport", "evaluate", "ece", "assert_regression",- "REPORT_SCHEMA", "questions_fingerprint", "file_fingerprint"):+ "REPORT_SCHEMA", "questions_fingerprint", "file_fingerprint",+ "brier", "aurc", "selective_accuracy", "is_confidence_metric"):check_true("laya.evals.%s exists" % name, hasattr(evals, name))check("REPORT_SCHEMA", evals.REPORT_SCHEMA, "laya-evals-report/1")