Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix: include zero confidence in ECE (#39)
Rajin Khan4e01745e01dd
diff --git a/laya/common.py b/laya/common.pyindex 2d030002..950c41df 100644--- a/laya/common.py+++ b/laya/common.py@@ -200,8 +200,8 @@ def ece_score(conf: np.ndarray, correct: np.ndarray, bins: int = 15) -> float:return float("nan")edges = np.linspace(0, 1, bins + 1)e = 0.0- for lo, hi in zip(edges[:-1], edges[1:]):- sel = (conf > lo) & (conf <= hi)+ for i, (lo, hi) in enumerate(zip(edges[:-1], edges[1:])):+ sel = (conf >= lo if i == 0 else conf > lo) & (conf <= hi)if sel.any():e += sel.mean() * abs(conf[sel].mean() - correct[sel].mean())return float(e)diff --git a/research/scripts/bench_local.py b/research/scripts/bench_local.pyindex 1c77dfdf..1b94ab20 100644--- a/research/scripts/bench_local.py+++ b/research/scripts/bench_local.py@@ -112,8 +112,8 @@ def ece_score(conf, corr, bins=15):if not len(conf):return float("nan")e, edges = 0.0, np.linspace(0, 1, bins + 1)- for lo, hi in zip(edges[:-1], edges[1:]):- s = (conf > lo) & (conf <= hi)+ for i, (lo, hi) in enumerate(zip(edges[:-1], edges[1:])):+ s = (conf >= lo if i == 0 else conf > lo) & (conf <= hi)if s.any():e += s.mean() * abs(conf[s].mean() - corr[s].mean())return float(e)diff --git a/research/scripts/build_benchmark_nb.py b/research/scripts/build_benchmark_nb.pyindex 8b131f2d..5c859035 100644--- a/research/scripts/build_benchmark_nb.py+++ b/research/scripts/build_benchmark_nb.py@@ -228,8 +228,8 @@ def ece_score(conf, correct, bins=15):return float("nan")edges = np.linspace(0, 1, bins + 1)e = 0.0- for lo, hi in zip(edges[:-1], edges[1:]):- sel = (conf > lo) & (conf <= hi)+ for i, (lo, hi) in enumerate(zip(edges[:-1], edges[1:])):+ sel = (conf >= lo if i == 0 else conf > lo) & (conf <= hi)if sel.any():e += sel.mean() * abs(conf[sel].mean() - correct[sel].mean())return float(e)diff --git a/tests/test_criteria.py b/tests/test_criteria.pyindex 1dd49bb1..2ecd5135 100644--- a/tests/test_criteria.py+++ b/tests/test_criteria.py@@ -13,9 +13,11 @@ import jsonimport osimport sys+import numpy as np+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))-from laya.common import render_criterion, render_options # noqa: E402+from laya.common import ece_score, render_criterion, render_options # noqa: E402PASS, FAIL = [], []@@ -77,6 +79,13 @@ check("score/str level unchanged", out[1], "level 1: high")check("score/int level -> json", out[2], "level 2: 2")+# --------------------------------------------------------------- calibration boundaries+check("ece/zero confidence is included",+ ece_score(np.array([0.0]), np.array([1.0])), 1.0)+check("ece/zero confidence has its proper weight",+ ece_score(np.array([0.0, 1.0]), np.array([1.0, 1.0])), 0.5)++# --------------------------------------------------------------- unchanged behaviourcheck("noul/default false text", render_options({"t": "noul", "ins": "x", "crit": None})[0],"false: no, the statement does not hold")