Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(hooks): reject invalid reliability calibration evidence

xp4113f67093dc
diff --git a/docs/hooks/api.md b/docs/hooks/api.md
index b68a6085..11d5ae9c 100644
--- a/docs/hooks/api.md
+++ b/docs/hooks/api.md
@@ -178,6 +178,11 @@ Collect `soft_stability` with `accept=None, escalate=None` on labelled evaluatio
then pass those numeric values and correctness labels to `laya.hooks.calibrate_reliability`.
Only fit or replay probed, numeric scores; exclude unprobed reports with `soft_stability=None`
and keep their `decision=None`.
+The fitter requires equal, nonempty score and label sequences. Scores and both accuracy
+targets must be finite Python `int`/`float` values in `[0, 1]` (excluding `bool`); invalid
+lengths or values raise `ValueError`. Correctness labels must be Python `bool` values,
+otherwise it raises `TypeError`. It does not drop unmatched rows or coerce labels such
+as `0`/`1` or `"false"`.
It chooses cutoffs from the distinct input scores at their supplied precision; equal scores
stay in the same group. `ACCEPT` includes scores equal to `accept`, while `ESCALATE` requires
a score strictly below `escalate`. If no cutoff meets the requested accuracy, `accept=1.01`
diff --git a/laya/hooks.py b/laya/hooks.py
index a4f11ed7..1f536858 100644
--- a/laya/hooks.py
+++ b/laya/hooks.py
@@ -657,10 +657,28 @@ def calibrate_reliability(
If no cutoff reaches `target_accuracy`, accept is returned as 1.01 (accept nothing).
Cutoffs use the supplied scores without rounding, matching the hook's reported
`soft_stability` and runtime comparison. Equal scores are evaluated together.
+
+ Scores and correctness labels must have equal, nonzero lengths. Scores and
+ accuracy targets must be finite int/float values in [0, 1], excluding bool;
+ invalid lengths or values raise ValueError. Correctness labels must be bool
+ values or TypeError is raised. Invalid evidence is never filtered or coerced.
"""
- pairs = sorted(zip(soft_scores, correct))
- if not pairs:
+ if len(soft_scores) != len(correct):
+ raise ValueError("soft_scores and correct must have the same length")
+ if len(soft_scores) == 0:
raise ValueError("cannot calibrate reliability on empty scores/correct pairs")
+ for name, value in (("target_accuracy", target_accuracy), ("max_escalate_accuracy", max_escalate_accuracy)):
+ if (isinstance(value, bool) or not isinstance(value, (int, float))
+ or not (0.0 <= value <= 1.0) or not math.isfinite(value)):
+ raise ValueError("%s must be a finite number in [0, 1], got %r" % (name, value))
+ for i, score in enumerate(soft_scores):
+ if (isinstance(score, bool) or not isinstance(score, (int, float))
+ or not (0.0 <= score <= 1.0) or not math.isfinite(score)):
+ raise ValueError("soft_scores[%d] must be a finite number in [0, 1], got %r" % (i, score))
+ for i, label in enumerate(correct):
+ if not isinstance(label, bool):
+ raise TypeError("correct[%d] must be a bool, got %r" % (i, label))
+ pairs = sorted(zip(soft_scores, correct))
cuts = sorted({s for s, _ in pairs})
accept = 1.01
diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py
index 2556df5d..6b9aebac 100644
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@ -183,6 +183,25 @@ check("serialized calibration cuts preserve accepted membership",
[decide_reliability(score, *_precise_cuts) for score in _precise_scores],
["VERIFY", "ACCEPT"])
+# Invalid calibration evidence raises at the public boundary instead of fitting a gate.
+for _name, _scores, _labels, _kwargs, _error in (
+ ("empty", [], [], {}, ValueError),
+ ("unmatched labels", [0.9, 0.9], [True], {}, ValueError),
+ ("unmatched scores", [0.9], [True, False], {}, ValueError),
+ ("nonfinite score", [0.9, float("nan")], [True, False], {}, ValueError),
+ ("nonboolean labels", [0.9, 0.9], [2, 0], {}, TypeError),
+ ("invalid target", [0.9], [False], {"target_accuracy": -0.1}, ValueError),
+ ("invalid escalation target", [0.9], [False], {"max_escalate_accuracy": float("nan")}, ValueError),
+):
+ try:
+ calibrate_reliability(_scores, _labels, **_kwargs)
+ except _error:
+ check_true("calibrate_reliability rejects " + _name, True)
+ except Exception as _exc:
+ check_true("calibrate_reliability rejects " + _name, False, repr(_exc))
+ else:
+ check_true("calibrate_reliability rejects " + _name, False, "did not raise")
+
# --------------------------------------------------------------- predict surfaces
from laya.mcp.remote import RemoteRouter # noqa: E402
diff --git a/tests/test_option_stability.py b/tests/test_option_stability.py
index 1ec59348..e5eebab2 100644
--- a/tests/test_option_stability.py
+++ b/tests/test_option_stability.py
@@ -102,6 +102,52 @@ kept = [c for s, c in zip(scores, correct) if s >= acc]
check_true("calibrate/target accuracy reached", sum(kept) / len(kept) >= 0.90)
+# Invalid evidence must not fit a usable gate by silently losing rows (#1312).
+complete_cuts = calibrate_reliability([0.9, 0.9], [True, False])
+check("calibrate/complete tied evidence accepts nothing", complete_cuts, (1.01, 0.0))
+check("calibrate/complete tied evidence replays as VERIFY",
+ decide_reliability(0.9, *complete_cuts), "VERIFY")
+for name, values, labels in (
+ ("short labels", [0.9, 0.9], [True]),
+ ("short scores", [0.9], [True, False]),
+ ("empty labels", [0.9], []),
+ ("empty scores", [], [True]),
+ ("NaN first", [float("nan"), 0.9], [False, True]),
+ ("NaN last", [0.9, float("nan")], [True, False]),
+):
+ check_raises("calibrate/rejects " + name, ValueError,
+ lambda: calibrate_reliability(values, labels))
+
+invalid_probabilities = (
+ -0.1, 1.01, float("nan"), float("inf"), -float("inf"),
+ True, False, "0.9", None, 0.9 + 0j, 10 ** 1000,
+)
+for value in invalid_probabilities:
+ check_raises("calibrate/rejects score %r" % (value,), ValueError,
+ lambda: calibrate_reliability([0.9, value], [True, False]))
+ for parameter in ("target_accuracy", "max_escalate_accuracy"):
+ check_raises("calibrate/rejects %s=%r" % (parameter, value), ValueError,
+ lambda: calibrate_reliability([0.9], [False], **{parameter: value}))
+
+for label in (0, 1, 2, 0.0, 1.0, "false", None):
+ check_raises("calibrate/rejects non-boolean label %r" % (label,), TypeError,
+ lambda: calibrate_reliability([0.9, 0.9], [True, label]))
+
+# Integer endpoints and tuple sequences remain valid; fitting must not mutate callers.
+for target in (0, 1):
+ for max_escalate in (0, 1):
+ cuts = calibrate_reliability((0, 1), (False, True), target, max_escalate)
+ check("calibrate/target=%d max_escalate=%d endpoints" % (target, max_escalate),
+ cuts, (target, 0.0))
+ check("calibrate/target=%d max_escalate=%d replay" % (target, max_escalate),
+ [decide_reliability(score, *cuts) for score in (0, 1)],
+ ["ACCEPT", "ACCEPT"] if target == 0 else ["VERIFY", "ACCEPT"])
+unsorted_scores, unsorted_labels = [0.9, 0.1, 0.5], [True, False, False]
+check("calibrate/unsorted evidence", calibrate_reliability(unsorted_scores, unsorted_labels), (0.9, 0.5))
+check("calibrate/does not mutate evidence", (unsorted_scores, unsorted_labels),
+ ([0.9, 0.1, 0.5], [True, False, False]))
+
+
# --------------------------------------------------------------- hook lifecycle: single state
hook = OptionStabilityHook(accept=0.77, escalate=0.61, seed="fixed-seed")
q_choice = {