Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

hooks: calibrate_reliability accepts NumPy booleans and reals after #1314

Nandakishore7249b79d4c5
diff --git a/laya/hooks.py b/laya/hooks.py
index 1f536858..81715f62 100644
--- a/laya/hooks.py
+++ b/laya/hooks.py
@@ -12,6 +12,7 @@ import asyncio
import contextvars
import inspect
import math
+import numbers
import random
import string
import threading
@@ -19,6 +20,8 @@ import time
import uuid
import warnings
import zlib
+import numpy as np
+
from contextlib import contextmanager
from dataclasses import dataclass, field
from typing import Any, Awaitable, Callable, Dict, List, Optional, Protocol, Sequence, Tuple, Union
@@ -659,24 +662,31 @@ def calibrate_reliability(
`soft_stability` and runtime comparison. Equal scores are evaluated together.
Scores and correctness labels must have equal, nonzero lengths. Scores and
- accuracy targets must be finite int/float values in [0, 1], excluding bool;
- invalid lengths or values raise ValueError. Correctness labels must be bool
- values or TypeError is raised. Invalid evidence is never filtered or coerced.
+ accuracy targets must be finite real numbers in [0, 1], excluding bool;
+ invalid lengths or values raise ValueError. Correctness labels must be
+ booleans or TypeError is raised. Invalid evidence is never filtered or coerced.
+
+ "Real number" and "boolean" mean the NumPy scalars too, because `correct` is
+ normally computed, not written out: `preds == gold` on a NumPy or pandas column
+ yields `np.bool_`, which is a boolean by every meaning except `isinstance(x, bool)`.
+ `numbers.Real` covers `np.float32` and `np.int64` and still excludes `np.bool_`,
+ so the two checks stay disjoint. `soft_scores` and `correct` must be sized; a
+ generator raises TypeError, since a length mismatch has to be detectable.
"""
if len(soft_scores) != len(correct):
raise ValueError("soft_scores and correct must have the same length")
if len(soft_scores) == 0:
raise ValueError("cannot calibrate reliability on empty scores/correct pairs")
for name, value in (("target_accuracy", target_accuracy), ("max_escalate_accuracy", max_escalate_accuracy)):
- if (isinstance(value, bool) or not isinstance(value, (int, float))
+ if (isinstance(value, (bool, np.bool_)) or not isinstance(value, numbers.Real)
or not (0.0 <= value <= 1.0) or not math.isfinite(value)):
raise ValueError("%s must be a finite number in [0, 1], got %r" % (name, value))
for i, score in enumerate(soft_scores):
- if (isinstance(score, bool) or not isinstance(score, (int, float))
+ if (isinstance(score, (bool, np.bool_)) or not isinstance(score, numbers.Real)
or not (0.0 <= score <= 1.0) or not math.isfinite(score)):
raise ValueError("soft_scores[%d] must be a finite number in [0, 1], got %r" % (i, score))
for i, label in enumerate(correct):
- if not isinstance(label, bool):
+ if not isinstance(label, (bool, np.bool_)):
raise TypeError("correct[%d] must be a bool, got %r" % (i, label))
pairs = sorted(zip(soft_scores, correct))
cuts = sorted({s for s, _ in pairs})
@@ -695,7 +705,9 @@ def calibrate_reliability(
below = [c for s, c in pairs if s < t]
if below and sum(below) / len(below) <= max_escalate_accuracy:
escalate = t
- return accept, escalate
+ # The cutoffs are thresholds a caller stores and replays, so hand back plain floats: a
+ # `np.float32` score would otherwise travel into a JSON report as a NumPy scalar.
+ return float(accept), float(escalate)
def _seed_of(state: Any) -> str:
diff --git a/tests/test_option_stability.py b/tests/test_option_stability.py
index e5eebab2..85a8a7b7 100644
--- a/tests/test_option_stability.py
+++ b/tests/test_option_stability.py
@@ -133,6 +133,37 @@ for label in (0, 1, 2, 0.0, 1.0, "false", None):
check_raises("calibrate/rejects non-boolean label %r" % (label,), TypeError,
lambda: calibrate_reliability([0.9, 0.9], [True, label]))
+# ... but a NumPy boolean IS a boolean, and `correct` is normally computed rather than written
+# out: `preds == gold` on a NumPy array or a pandas column yields `np.bool_`, which fails a bare
+# `isinstance(x, bool)`. Same for `np.float32`/`np.int64` scores, which `numbers.Real` accepts
+# while still excluding `np.bool_`, so the score and label checks stay disjoint. The fitted
+# cutoffs come back as plain floats either way, because a caller stores and replays them.
+import numpy as _np # noqa: E402
+
+_want = calibrate_reliability([0.9, 0.4, 0.8], [True, False, True], target_accuracy=1.0)
+check("calibrate/plain floats and bools", _want, (0.8, 0.0))
+for _name, _scores, _labels in (
+ ("numpy bool array labels", [0.9, 0.4, 0.8], _np.array([True, False, True])),
+ ("numpy bool scalars", [0.9, 0.4, 0.8], list(_np.array([1, 0, 1]) == 1)),
+ ("numpy float32 scores", list(_np.array([0.9, 0.4, 0.8], dtype=_np.float32)), [True, False, True]),
+ ("numpy float64 scores", list(_np.array([0.9, 0.4, 0.8])), [True, False, True]),
+ ("numpy int scores", list(_np.array([1, 0, 1], dtype=_np.int64)), [True, False, True]),
+):
+ _got = calibrate_reliability(_scores, _labels, target_accuracy=1.0)
+ check_true("calibrate/%s fits" % _name, _got[0] <= 1.0 and _got[1] == 0.0, repr(_got))
+ check("calibrate/%s returns plain floats" % _name,
+ [type(_c).__name__ for _c in _got], ["float", "float"])
+check("calibrate/numpy bools fit the same cutoffs as python bools",
+ calibrate_reliability([0.9, 0.4, 0.8], _np.array([True, False, True]), target_accuracy=1.0), _want)
+# A NumPy boolean is still not a score, and a NumPy float is still not a label.
+check_raises("calibrate/rejects numpy bool score", ValueError,
+ lambda: calibrate_reliability([_np.True_], [True]))
+check_raises("calibrate/rejects numpy float label", TypeError,
+ lambda: calibrate_reliability([0.9], [_np.float64(1.0)]))
+# Sized sequences only: a length mismatch has to be detectable, so a generator is refused.
+check_raises("calibrate/rejects a generator of scores", TypeError,
+ lambda: calibrate_reliability((x for x in [0.9]), [True]))
+
# Integer endpoints and tuple sequences remain valid; fitting must not mutate callers.
for target in (0, 1):
for max_escalate in (0, 1):