Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix: include zero confidence in ECE (#39)

Rajin Khan4e01745e01dd
diff --git a/laya/common.py b/laya/common.py
index 2d030002..950c41df 100644
--- a/laya/common.py
+++ b/laya/common.py
@@ -200,8 +200,8 @@ def ece_score(conf: np.ndarray, correct: np.ndarray, bins: int = 15) -> float:
return float("nan")
edges = np.linspace(0, 1, bins + 1)
e = 0.0
- for lo, hi in zip(edges[:-1], edges[1:]):
- sel = (conf > lo) & (conf <= hi)
+ for i, (lo, hi) in enumerate(zip(edges[:-1], edges[1:])):
+ sel = (conf >= lo if i == 0 else conf > lo) & (conf <= hi)
if sel.any():
e += sel.mean() * abs(conf[sel].mean() - correct[sel].mean())
return float(e)
diff --git a/research/scripts/bench_local.py b/research/scripts/bench_local.py
index 1c77dfdf..1b94ab20 100644
--- a/research/scripts/bench_local.py
+++ b/research/scripts/bench_local.py
@@ -112,8 +112,8 @@ def ece_score(conf, corr, bins=15):
if not len(conf):
return float("nan")
e, edges = 0.0, np.linspace(0, 1, bins + 1)
- for lo, hi in zip(edges[:-1], edges[1:]):
- s = (conf > lo) & (conf <= hi)
+ for i, (lo, hi) in enumerate(zip(edges[:-1], edges[1:])):
+ s = (conf >= lo if i == 0 else conf > lo) & (conf <= hi)
if s.any():
e += s.mean() * abs(conf[s].mean() - corr[s].mean())
return float(e)
diff --git a/research/scripts/build_benchmark_nb.py b/research/scripts/build_benchmark_nb.py
index 8b131f2d..5c859035 100644
--- a/research/scripts/build_benchmark_nb.py
+++ b/research/scripts/build_benchmark_nb.py
@@ -228,8 +228,8 @@ def ece_score(conf, correct, bins=15):
return float("nan")
edges = np.linspace(0, 1, bins + 1)
e = 0.0
- for lo, hi in zip(edges[:-1], edges[1:]):
- sel = (conf > lo) & (conf <= hi)
+ for i, (lo, hi) in enumerate(zip(edges[:-1], edges[1:])):
+ sel = (conf >= lo if i == 0 else conf > lo) & (conf <= hi)
if sel.any():
e += sel.mean() * abs(conf[sel].mean() - correct[sel].mean())
return float(e)
diff --git a/tests/test_criteria.py b/tests/test_criteria.py
index 1dd49bb1..2ecd5135 100644
--- a/tests/test_criteria.py
+++ b/tests/test_criteria.py
@@ -13,9 +13,11 @@ import json
import os
import sys
+import numpy as np
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-from laya.common import render_criterion, render_options # noqa: E402
+from laya.common import ece_score, render_criterion, render_options # noqa: E402
PASS, FAIL = [], []
@@ -77,6 +79,13 @@ check("score/str level unchanged", out[1], "level 1: high")
check("score/int level -> json", out[2], "level 2: 2")
+# --------------------------------------------------------------- calibration boundaries
+check("ece/zero confidence is included",
+ ece_score(np.array([0.0]), np.array([1.0])), 1.0)
+check("ece/zero confidence has its proper weight",
+ ece_score(np.array([0.0, 1.0]), np.array([1.0, 1.0])), 0.5)
+
+
# --------------------------------------------------------------- unchanged behaviour
check("noul/default false text", render_options({"t": "noul", "ins": "x", "crit": None})[0],
"false: no, the statement does not hold")