Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): use nearest-rank 95th percentile

rahul05ranjan81a07e57852f
diff --git a/laya/evals.py b/laya/evals.py
index 050d8c8f..e15d2848 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -290,8 +290,12 @@ def _group_cases(cases: Sequence[Dict[str, Any]], key: str) -> Dict[str, List[Di
def _percentiles(values: Sequence[float]) -> Tuple[float, float]:
"""(median, 95th) using the nearest-rank rule the report has always used for latency."""
ordered = sorted(values)
- return (float(statistics.median(ordered)),
- float(ordered[min(len(ordered) - 1, int(len(ordered) * 0.95))]))
+ n = len(ordered)
+ # Nearest-rank 95th percentile: the ceil(0.95 * n)-th smallest value (1-indexed).
+ # Integer arithmetic avoids the float rounding that made `int(n * 0.95)` return the
+ # (0.95n + 1)-th value whenever n is a multiple of 20.
+ rank = (95 * n + 99) // 100
+ return (float(statistics.median(ordered)), float(ordered[rank - 1]))
def _aggregate(cases: Sequence[Dict[str, Any]], evaluators: Sequence[Evaluator]) -> Dict[str, float]:
diff --git a/tests/test_evals.py b/tests/test_evals.py
index ce0db32a..79f3c223 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -92,6 +92,20 @@ def test_ece_on_known_inputs():
assert ece([], []) is None
+def test_percentiles_use_nearest_rank():
+ from laya.evals import _percentiles
+
+ # Nearest-rank 95th percentile is the ceil(0.95 * n)-th smallest value (1-indexed).
+ # `int(n * 0.95)` used to return the (0.95n + 1)-th value whenever n is a multiple of 20.
+ for n in (20, 40, 100):
+ values = list(range(1, n + 1))
+ _, p95 = _percentiles(values)
+ assert p95 == pytest.approx((95 * n + 99) // 100), \
+ "95th percentile of 1..%d must be the nearest-rank value, got %r" % (n, p95)
+ # A single value is its own percentile.
+ assert _percentiles([7.0]) == (7.0, 7.0)
+
+
# --------------------------------------------------------------- dataset
def test_dataset_from_jsonl(tmp_path):
path = tmp_path / "d.jsonl"