Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
hooks: calibrate_reliability accepts NumPy booleans and reals after #1314
Nandakishore7249b79d4c5
diff --git a/laya/hooks.py b/laya/hooks.pyindex 1f536858..81715f62 100644--- a/laya/hooks.py+++ b/laya/hooks.py@@ -12,6 +12,7 @@ import asyncioimport contextvarsimport inspectimport math+import numbersimport randomimport stringimport threading@@ -19,6 +20,8 @@ import timeimport uuidimport warningsimport zlib+import numpy as np+from contextlib import contextmanagerfrom dataclasses import dataclass, fieldfrom typing import Any, Awaitable, Callable, Dict, List, Optional, Protocol, Sequence, Tuple, Union@@ -659,24 +662,31 @@ def calibrate_reliability(`soft_stability` and runtime comparison. Equal scores are evaluated together.Scores and correctness labels must have equal, nonzero lengths. Scores and- accuracy targets must be finite int/float values in [0, 1], excluding bool;- invalid lengths or values raise ValueError. Correctness labels must be bool- values or TypeError is raised. Invalid evidence is never filtered or coerced.+ accuracy targets must be finite real numbers in [0, 1], excluding bool;+ invalid lengths or values raise ValueError. Correctness labels must be+ booleans or TypeError is raised. Invalid evidence is never filtered or coerced.++ "Real number" and "boolean" mean the NumPy scalars too, because `correct` is+ normally computed, not written out: `preds == gold` on a NumPy or pandas column+ yields `np.bool_`, which is a boolean by every meaning except `isinstance(x, bool)`.+ `numbers.Real` covers `np.float32` and `np.int64` and still excludes `np.bool_`,+ so the two checks stay disjoint. `soft_scores` and `correct` must be sized; a+ generator raises TypeError, since a length mismatch has to be detectable."""if len(soft_scores) != len(correct):raise ValueError("soft_scores and correct must have the same length")if len(soft_scores) == 0:raise ValueError("cannot calibrate reliability on empty scores/correct pairs")for name, value in (("target_accuracy", target_accuracy), ("max_escalate_accuracy", max_escalate_accuracy)):- if (isinstance(value, bool) or not isinstance(value, (int, float))+ if (isinstance(value, (bool, np.bool_)) or not isinstance(value, numbers.Real)or not (0.0 <= value <= 1.0) or not math.isfinite(value)):raise ValueError("%s must be a finite number in [0, 1], got %r" % (name, value))for i, score in enumerate(soft_scores):- if (isinstance(score, bool) or not isinstance(score, (int, float))+ if (isinstance(score, (bool, np.bool_)) or not isinstance(score, numbers.Real)or not (0.0 <= score <= 1.0) or not math.isfinite(score)):raise ValueError("soft_scores[%d] must be a finite number in [0, 1], got %r" % (i, score))for i, label in enumerate(correct):- if not isinstance(label, bool):+ if not isinstance(label, (bool, np.bool_)):raise TypeError("correct[%d] must be a bool, got %r" % (i, label))pairs = sorted(zip(soft_scores, correct))cuts = sorted({s for s, _ in pairs})@@ -695,7 +705,9 @@ def calibrate_reliability(below = [c for s, c in pairs if s < t]if below and sum(below) / len(below) <= max_escalate_accuracy:escalate = t- return accept, escalate+ # The cutoffs are thresholds a caller stores and replays, so hand back plain floats: a+ # `np.float32` score would otherwise travel into a JSON report as a NumPy scalar.+ return float(accept), float(escalate)def _seed_of(state: Any) -> str:diff --git a/tests/test_option_stability.py b/tests/test_option_stability.pyindex e5eebab2..85a8a7b7 100644--- a/tests/test_option_stability.py+++ b/tests/test_option_stability.py@@ -133,6 +133,37 @@ for label in (0, 1, 2, 0.0, 1.0, "false", None):check_raises("calibrate/rejects non-boolean label %r" % (label,), TypeError,lambda: calibrate_reliability([0.9, 0.9], [True, label]))+# ... but a NumPy boolean IS a boolean, and `correct` is normally computed rather than written+# out: `preds == gold` on a NumPy array or a pandas column yields `np.bool_`, which fails a bare+# `isinstance(x, bool)`. Same for `np.float32`/`np.int64` scores, which `numbers.Real` accepts+# while still excluding `np.bool_`, so the score and label checks stay disjoint. The fitted+# cutoffs come back as plain floats either way, because a caller stores and replays them.+import numpy as _np # noqa: E402++_want = calibrate_reliability([0.9, 0.4, 0.8], [True, False, True], target_accuracy=1.0)+check("calibrate/plain floats and bools", _want, (0.8, 0.0))+for _name, _scores, _labels in (+ ("numpy bool array labels", [0.9, 0.4, 0.8], _np.array([True, False, True])),+ ("numpy bool scalars", [0.9, 0.4, 0.8], list(_np.array([1, 0, 1]) == 1)),+ ("numpy float32 scores", list(_np.array([0.9, 0.4, 0.8], dtype=_np.float32)), [True, False, True]),+ ("numpy float64 scores", list(_np.array([0.9, 0.4, 0.8])), [True, False, True]),+ ("numpy int scores", list(_np.array([1, 0, 1], dtype=_np.int64)), [True, False, True]),+):+ _got = calibrate_reliability(_scores, _labels, target_accuracy=1.0)+ check_true("calibrate/%s fits" % _name, _got[0] <= 1.0 and _got[1] == 0.0, repr(_got))+ check("calibrate/%s returns plain floats" % _name,+ [type(_c).__name__ for _c in _got], ["float", "float"])+check("calibrate/numpy bools fit the same cutoffs as python bools",+ calibrate_reliability([0.9, 0.4, 0.8], _np.array([True, False, True]), target_accuracy=1.0), _want)+# A NumPy boolean is still not a score, and a NumPy float is still not a label.+check_raises("calibrate/rejects numpy bool score", ValueError,+ lambda: calibrate_reliability([_np.True_], [True]))+check_raises("calibrate/rejects numpy float label", TypeError,+ lambda: calibrate_reliability([0.9], [_np.float64(1.0)]))+# Sized sequences only: a length mismatch has to be detectable, so a generator is refused.+check_raises("calibrate/rejects a generator of scores", TypeError,+ lambda: calibrate_reliability((x for x in [0.9]), [True]))+# Integer endpoints and tuple sequences remain valid; fitting must not mutate callers.for target in (0, 1):for max_escalate in (0, 1):