Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(confidence): refuse a negative probability instead of a Jev confidence outside [0, 1]

Aashish12dca7e3ea6e
diff --git a/laya/confidence.py b/laya/confidence.py
index c7530d13..cd170959 100644
--- a/laya/confidence.py
+++ b/laya/confidence.py
@@ -46,8 +46,14 @@ def jev_confidence(answer: Dict[str, Any]) -> Optional[float]:
score max(0, 1 - sum(p_i * |i - mode|) / D), D the same sum for a uniform distribution
Both are 1.0 for a single option, and a zero total is read as uniform, as the adapter does.
- A noul answer has no Jev `confidence`, and malformed probabilities give None rather than a
- guess, so a caller filtering on this number never filters on a made-up one.
+ A noul answer has no Jev `confidence`, and a malformed document -- a non-numeric, boolean,
+ non-finite or negative probability -- gives None rather than a guess, so a caller filtering on
+ this number never filters on a made-up one. Negatives are the case that needs naming: the
+ division below normalises by the sum, so a mixed-sign document can divide by a near-zero or
+ negative total and return a number outside [0, 1] for a distribution whose argmax is not the
+ one it measured, while an all-negative document divides back into range and returns exactly
+ what its positive counterpart returns. Refusing them is what makes the range a property: every
+ number this returns lies in [0, 1].
"""
kind = answer.get("type")
probabilities = answer.get("probabilities")
@@ -60,7 +66,8 @@ def jev_confidence(answer: Dict[str, Any]) -> Optional[float]:
p = [float(value) for value in probabilities.values()]
except (TypeError, ValueError):
return None
- if any(isinstance(v, bool) for v in probabilities.values()) or not all(math.isfinite(v) for v in p):
+ if (any(isinstance(v, bool) for v in probabilities.values())
+ or not all(math.isfinite(v) and v >= 0.0 for v in p)):
return None
n = len(p)
if n == 1:
diff --git a/tests/test_confidence.py b/tests/test_confidence.py
index a424d8c7..79d629b9 100644
--- a/tests/test_confidence.py
+++ b/tests/test_confidence.py
@@ -2465,9 +2465,40 @@ for _name, _answer in (
("bool-probability", _choice(True, False)),
("nan-probability", _choice(float("nan"), 0.5)),
("non-integer-score-level", {"type": "score", "probabilities": {"low": 0.5, "high": 0.5}}),
+ # A negative probability is malformed, and at `432312b3` none of these five were refused: they
+ # returned 3.0, 2.0, 0.8, 121.0 (120.99999999999989) and 1.75. The third is the one that cannot
+ # be caught by inspection -- an all-negative document divides by its own negative total and
+ # comes back with the number its positive counterpart comes back with.
+ ("negative-probability", _choice(2.0, -1.0)),
+ ("mixed-sign-argmax-flips", _choice(0.5, -1.5)),
+ ("all-negative-probabilities", _choice(-0.9, -0.1)),
+ ("negative-total-near-zero", _choice(-0.61, 0.6)),
+ ("negative-score-level", _score(1.5, -0.5, 0.0)),
):
check("jev_confidence/none-for-%s" % _name, jev_confidence(_answer), None)
+# The range the name promises: nothing this returns is outside [0, 1], and the negatives are
+# exactly what is refused. `x_jev_confidence` is served for a client to compare a threshold
+# against, so an out-of-range number is a gate that passes every answer it touches.
+_swept, _out_of_range, _refused = 0, [], []
+for _i in range(-200, 301):
+ for _answer in (_choice(_i / 100.0, 0.6), _score(_i / 100.0, 0.4, 0.2)):
+ _got = jev_confidence(_answer)
+ _swept += 1
+ if _got is None:
+ _refused.append(_i / 100.0)
+ elif not (0.0 <= _got <= 1.0):
+ _out_of_range.append((_i / 100.0, _got))
+check_true("jev_confidence/sweep kept every number inside [0, 1]", not _out_of_range,
+ "%d of %d outside, worst %r" % (len(_out_of_range), _swept, _out_of_range[:3]))
+check_true("jev_confidence/sweep refused exactly the negative probability",
+ sorted(set(_refused)) == [value / 100.0 for value in range(-200, 0)],
+ "%d refused: %r" % (len(_refused), _refused[:2] + _refused[-2:]))
+# 501 values x both answer types, and the 400 refusals are the 200 negatives twice over
+check_true("jev_confidence/sweep measured everything that is not negative",
+ (_swept, len(_refused), _swept - len(_refused)) == (1002, 400, 602),
+ "swept %d, refused %d, measured %d" % (_swept, len(_refused), _swept - len(_refused)))
+
# --- the public name and the served field must be one computation (#302) --------------------
# `/v1/systemone` publishes `x_jev_confidence`, but a caller running the checkpoint in-process
# gets an answer dict and no HTTP layer, and the other two confidence quantities