Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(hooks): reject invalid reliability calibration evidence
xp4113f67093dc
diff --git a/docs/hooks/api.md b/docs/hooks/api.mdindex b68a6085..11d5ae9c 100644--- a/docs/hooks/api.md+++ b/docs/hooks/api.md@@ -178,6 +178,11 @@ Collect `soft_stability` with `accept=None, escalate=None` on labelled evaluatiothen pass those numeric values and correctness labels to `laya.hooks.calibrate_reliability`.Only fit or replay probed, numeric scores; exclude unprobed reports with `soft_stability=None`and keep their `decision=None`.+The fitter requires equal, nonempty score and label sequences. Scores and both accuracy+targets must be finite Python `int`/`float` values in `[0, 1]` (excluding `bool`); invalid+lengths or values raise `ValueError`. Correctness labels must be Python `bool` values,+otherwise it raises `TypeError`. It does not drop unmatched rows or coerce labels such+as `0`/`1` or `"false"`.It chooses cutoffs from the distinct input scores at their supplied precision; equal scoresstay in the same group. `ACCEPT` includes scores equal to `accept`, while `ESCALATE` requiresa score strictly below `escalate`. If no cutoff meets the requested accuracy, `accept=1.01`diff --git a/laya/hooks.py b/laya/hooks.pyindex a4f11ed7..1f536858 100644--- a/laya/hooks.py+++ b/laya/hooks.py@@ -657,10 +657,28 @@ def calibrate_reliability(If no cutoff reaches `target_accuracy`, accept is returned as 1.01 (accept nothing).Cutoffs use the supplied scores without rounding, matching the hook's reported`soft_stability` and runtime comparison. Equal scores are evaluated together.++ Scores and correctness labels must have equal, nonzero lengths. Scores and+ accuracy targets must be finite int/float values in [0, 1], excluding bool;+ invalid lengths or values raise ValueError. Correctness labels must be bool+ values or TypeError is raised. Invalid evidence is never filtered or coerced."""- pairs = sorted(zip(soft_scores, correct))- if not pairs:+ if len(soft_scores) != len(correct):+ raise ValueError("soft_scores and correct must have the same length")+ if len(soft_scores) == 0:raise ValueError("cannot calibrate reliability on empty scores/correct pairs")+ for name, value in (("target_accuracy", target_accuracy), ("max_escalate_accuracy", max_escalate_accuracy)):+ if (isinstance(value, bool) or not isinstance(value, (int, float))+ or not (0.0 <= value <= 1.0) or not math.isfinite(value)):+ raise ValueError("%s must be a finite number in [0, 1], got %r" % (name, value))+ for i, score in enumerate(soft_scores):+ if (isinstance(score, bool) or not isinstance(score, (int, float))+ or not (0.0 <= score <= 1.0) or not math.isfinite(score)):+ raise ValueError("soft_scores[%d] must be a finite number in [0, 1], got %r" % (i, score))+ for i, label in enumerate(correct):+ if not isinstance(label, bool):+ raise TypeError("correct[%d] must be a bool, got %r" % (i, label))+ pairs = sorted(zip(soft_scores, correct))cuts = sorted({s for s, _ in pairs})accept = 1.01diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.pyindex 2556df5d..6b9aebac 100644--- a/tests/test_hooks_api.py+++ b/tests/test_hooks_api.py@@ -183,6 +183,25 @@ check("serialized calibration cuts preserve accepted membership",[decide_reliability(score, *_precise_cuts) for score in _precise_scores],["VERIFY", "ACCEPT"])+# Invalid calibration evidence raises at the public boundary instead of fitting a gate.+for _name, _scores, _labels, _kwargs, _error in (+ ("empty", [], [], {}, ValueError),+ ("unmatched labels", [0.9, 0.9], [True], {}, ValueError),+ ("unmatched scores", [0.9], [True, False], {}, ValueError),+ ("nonfinite score", [0.9, float("nan")], [True, False], {}, ValueError),+ ("nonboolean labels", [0.9, 0.9], [2, 0], {}, TypeError),+ ("invalid target", [0.9], [False], {"target_accuracy": -0.1}, ValueError),+ ("invalid escalation target", [0.9], [False], {"max_escalate_accuracy": float("nan")}, ValueError),+):+ try:+ calibrate_reliability(_scores, _labels, **_kwargs)+ except _error:+ check_true("calibrate_reliability rejects " + _name, True)+ except Exception as _exc:+ check_true("calibrate_reliability rejects " + _name, False, repr(_exc))+ else:+ check_true("calibrate_reliability rejects " + _name, False, "did not raise")+# --------------------------------------------------------------- predict surfacesfrom laya.mcp.remote import RemoteRouter # noqa: E402diff --git a/tests/test_option_stability.py b/tests/test_option_stability.pyindex 1ec59348..e5eebab2 100644--- a/tests/test_option_stability.py+++ b/tests/test_option_stability.py@@ -102,6 +102,52 @@ kept = [c for s, c in zip(scores, correct) if s >= acc]check_true("calibrate/target accuracy reached", sum(kept) / len(kept) >= 0.90)+# Invalid evidence must not fit a usable gate by silently losing rows (#1312).+complete_cuts = calibrate_reliability([0.9, 0.9], [True, False])+check("calibrate/complete tied evidence accepts nothing", complete_cuts, (1.01, 0.0))+check("calibrate/complete tied evidence replays as VERIFY",+ decide_reliability(0.9, *complete_cuts), "VERIFY")+for name, values, labels in (+ ("short labels", [0.9, 0.9], [True]),+ ("short scores", [0.9], [True, False]),+ ("empty labels", [0.9], []),+ ("empty scores", [], [True]),+ ("NaN first", [float("nan"), 0.9], [False, True]),+ ("NaN last", [0.9, float("nan")], [True, False]),+):+ check_raises("calibrate/rejects " + name, ValueError,+ lambda: calibrate_reliability(values, labels))++invalid_probabilities = (+ -0.1, 1.01, float("nan"), float("inf"), -float("inf"),+ True, False, "0.9", None, 0.9 + 0j, 10 ** 1000,+)+for value in invalid_probabilities:+ check_raises("calibrate/rejects score %r" % (value,), ValueError,+ lambda: calibrate_reliability([0.9, value], [True, False]))+ for parameter in ("target_accuracy", "max_escalate_accuracy"):+ check_raises("calibrate/rejects %s=%r" % (parameter, value), ValueError,+ lambda: calibrate_reliability([0.9], [False], **{parameter: value}))++for label in (0, 1, 2, 0.0, 1.0, "false", None):+ check_raises("calibrate/rejects non-boolean label %r" % (label,), TypeError,+ lambda: calibrate_reliability([0.9, 0.9], [True, label]))++# Integer endpoints and tuple sequences remain valid; fitting must not mutate callers.+for target in (0, 1):+ for max_escalate in (0, 1):+ cuts = calibrate_reliability((0, 1), (False, True), target, max_escalate)+ check("calibrate/target=%d max_escalate=%d endpoints" % (target, max_escalate),+ cuts, (target, 0.0))+ check("calibrate/target=%d max_escalate=%d replay" % (target, max_escalate),+ [decide_reliability(score, *cuts) for score in (0, 1)],+ ["ACCEPT", "ACCEPT"] if target == 0 else ["VERIFY", "ACCEPT"])+unsorted_scores, unsorted_labels = [0.9, 0.1, 0.5], [True, False, False]+check("calibrate/unsorted evidence", calibrate_reliability(unsorted_scores, unsorted_labels), (0.9, 0.5))+check("calibrate/does not mutate evidence", (unsorted_scores, unsorted_labels),+ ([0.9, 0.1, 0.5], [True, False, False]))++# --------------------------------------------------------------- hook lifecycle: single statehook = OptionStabilityHook(accept=0.77, escalate=0.61, seed="fixed-seed")q_choice = {