Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): refuse non-finite act evidence in the act-head diagnostic

Bruce-Yiibf1046561528
diff --git a/research/evals/act_head_eval.py b/research/evals/act_head_eval.py
index a216ab96..44963b3a 100644
--- a/research/evals/act_head_eval.py
+++ b/research/evals/act_head_eval.py
@@ -29,8 +29,8 @@ denominator:
|---|---|
| `paired` | usable for both AUCs |
| `no_binary_correctness` | `correct` is not a bool -- a `score` answer, where correctness is not unambiguous |
-| `missing_act_probability` | `action.act_probability` absent or not a real number |
-| `missing_confidence` | `correct` and `act_probability` fine, but no numeric `confidence` |
+| `missing_act_probability` | `action.act_probability` absent, not a number, or not finite |
+| `missing_confidence` | `correct` and `act_probability` fine, but `confidence` absent, not a number, or not finite |
| `malformed_case` | the entry is not a case object at all |
The `paired` set is used for **both** the `act_probability` AUC and the `confidence`
@@ -57,12 +57,20 @@ ROC-AUC is undefined when one class is absent, and this harness reports `null` w
maximum and unique-value count plus a `saturated` flag: a signal pinned at one value is a
failed diagnostic, not a 0.5 result to be averaged away.
+A number is only accepted when it is finite. `json.load` reads the non-standard `NaN` /
+`Infinity` / `-Infinity` tokens by default, so a malformed report can carry them in, and
+ranking one returns a confident-looking number that means nothing -- 0.5 for `NaN` and
+`+inf`, 0.0 for `-inf`, where 0.0 is easily misread as a real anti-correlated signal. They
+are bucketed as missing evidence instead, `roc_auc` raises `ValueError` if called directly
+with one, and the JSON writer sets `allow_nan=False` so none can leave this tool.
+
python research/evals/act_head_eval.py report.json [--json out.json] [--markdown out.md]
"""
from __future__ import annotations
import argparse
import json
+import math
import sys
from typing import Any, Dict, List, Optional, Sequence
@@ -79,8 +87,16 @@ _NO_CASES = "no scorable cases"
def _is_number(value: Any) -> bool:
- """True for a real number. `bool` is an `int` subclass, and is not a measurement."""
- return isinstance(value, (int, float)) and not isinstance(value, bool)
+ """True for a finite real number.
+
+ ``bool`` is an ``int`` subclass and is not a measurement. NaN and the infinities are
+ excluded too: ``json.load`` accepts the non-standard ``NaN`` / ``Infinity`` /
+ ``-Infinity`` tokens, so a malformed report can carry them in, and they are not
+ evidence of anything.
+ """
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
+ return False
+ return math.isfinite(float(value))
def roc_auc(scores: Sequence[float], labels: Sequence[bool]) -> Optional[float]:
@@ -88,11 +104,19 @@ def roc_auc(scores: Sequence[float], labels: Sequence[bool]) -> Optional[float]:
None when either class is absent, or when there is nothing to score: ROC-AUC needs
both a positive and a negative case to be defined at all.
+
+ Raises ValueError on a non-finite score. This function is callable on its own, and
+ ranking NaN or an infinity yields a confident-looking number -- 0.5 for NaN and for
+ ``+inf``, 0.0 for ``-inf`` -- where 0.0 reads as "the signal is perfectly
+ anti-correlated" rather than "the input was not a measurement". Refusing is the only
+ honest answer.
"""
y = np.asarray(labels, dtype=bool)
s = np.asarray(scores, dtype=float)
if y.shape != s.shape:
raise ValueError("scores and labels must be the same length")
+ if not np.all(np.isfinite(s)):
+ raise ValueError("scores must all be finite numbers")
n_pos = int(y.sum())
n_neg = int(y.shape[0] - n_pos)
if n_pos == 0 or n_neg == 0:
@@ -274,7 +298,10 @@ def main(argv: Optional[List[str]] = None) -> int:
if args.json_out:
with open(args.json_out, "w", encoding="utf-8") as handle:
- json.dump(result, handle, indent=2, sort_keys=True)
+ # allow_nan=False: this diagnostic refuses non-finite evidence on the way in,
+ # so a NaN reaching the output would be a bug worth failing on, not a value
+ # to write out as non-standard JSON.
+ json.dump(result, handle, indent=2, sort_keys=True, allow_nan=False)
handle.write("\n")
if args.md_out:
with open(args.md_out, "w", encoding="utf-8") as handle:
diff --git a/research/evals/test_act_head_eval.py b/research/evals/test_act_head_eval.py
index 8d982dc7..88b51b8b 100644
--- a/research/evals/test_act_head_eval.py
+++ b/research/evals/test_act_head_eval.py
@@ -180,6 +180,56 @@ junk = analyse_cases([None, "nope", 5, case(0.9, True, 0.8)])
check("analyse/junk_counted", junk["skipped"]["malformed_case"], 3)
check("analyse/junk_paired", junk["paired"], 1)
+# ------------------------------------------------------------------ non-finite evidence
+# `json.load` accepts the non-standard NaN / Infinity / -Infinity tokens by default, so a
+# malformed report can carry them into the harness. They are not measurements: ranking
+# them produced a confident-looking 0.5 for NaN and +Inf and 0.0 for -Inf, where 0.0 reads
+# as "the signal is perfectly anti-correlated" rather than "the input was garbage".
+NAN, INF, NEG_INF = float("nan"), float("inf"), float("-inf")
+
+for label, value in (("nan", NAN), ("posinf", INF), ("neginf", NEG_INF)):
+ nonfinite_act = analyse_cases([
+ case(0.9, True, 0.8),
+ case(value, False, 0.3),
+ case(0.1, False, 0.2),
+ ])
+ check("analyse/nonfinite_act_counted_%s" % label,
+ nonfinite_act["skipped"]["missing_act_probability"], 1)
+ check("analyse/nonfinite_act_paired_%s" % label, nonfinite_act["paired"], 2)
+ check("analyse/nonfinite_act_no_fake_auc_%s" % label, nonfinite_act["act_auc"], 1.0)
+ check("analyse/nonfinite_act_excluded_from_spread_%s" % label,
+ (nonfinite_act["act_min"], nonfinite_act["act_max"]), (0.1, 0.9))
+
+ nonfinite_conf = analyse_cases([
+ case(0.9, True, 0.8),
+ case(0.2, False, value),
+ case(0.1, False, 0.2),
+ ])
+ check("analyse/nonfinite_confidence_counted_%s" % label,
+ nonfinite_conf["skipped"]["missing_confidence"], 1)
+ check("analyse/nonfinite_confidence_paired_%s" % label, nonfinite_conf["paired"], 2)
+ # The dropped case is the middle one; the two survivors still separate cleanly, and
+ # the act AUC must be computed on exactly that reduced set.
+ check("analyse/nonfinite_confidence_auc_%s" % label, nonfinite_conf["act_auc"], 1.0)
+
+# `roc_auc` is independently callable, so it rejects non-finite scores itself rather than
+# relying on the caller having filtered them.
+for label, value in (("nan", NAN), ("posinf", INF), ("neginf", NEG_INF)):
+ try:
+ roc_auc([value, 0.5, 0.9], [True, False, True])
+ FAIL.append("roc_auc should reject a %s score" % label)
+ except ValueError as exc:
+ check_true("roc_auc/rejects_%s" % label, "finite" in str(exc).lower(), str(exc))
+
+# A report that is entirely non-finite leaves nothing to score, which is a clean null
+# state rather than a fabricated number.
+all_bad = analyse_cases([case(NAN, True, 0.8), case(0.5, False, INF)])
+check("analyse/all_nonfinite_paired", all_bad["paired"], 0)
+check("analyse/all_nonfinite_auc", all_bad["act_auc"], None)
+check("analyse/all_nonfinite_reason", all_bad["auc_null_reason"], "no scorable cases")
+check("analyse/all_nonfinite_counted",
+ all_bad["skipped"]["missing_confidence"] + all_bad["skipped"]["missing_act_probability"], 2)
+
# ------------------------------------------------------------------ report plumbing
REPORT = {
"config": {"model": "english"},