Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
"""Tests for OptionStabilityHook and metamorphic reliability signal (#635).
Pure Python tests: runs offline without torch or checkpoint downloads.
Run: python tests/test_option_stability.py
"""
import json
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from laya.hooks import ( # noqa: E402
ALL_VARIANTS,
OptionStabilityHook,
PredictContext,
VARIANT_PRESETS,
calibrate_reliability,
decide_reliability,
make_variants,
)
PASS, FAIL = [], []
def check(name, got, want):
if got == want:
PASS.append(name)
else:
FAIL.append("%s: got %r, want %r" % (name, got, want))
def check_true(name, cond, detail=""):
if cond:
PASS.append(name)
else:
FAIL.append("%s%s" % (name, ": " + detail if detail else ""))
def check_raises(name, exc, fn):
try:
fn()
except exc:
PASS.append(name)
return
except BaseException as other: # noqa: BLE001
FAIL.append("%s: raised %r, want %r" % (name, other, exc))
return
FAIL.append("%s: did not raise %r" % (name, exc))
# --------------------------------------------------------------- constants & presets
check("constants/ALL_VARIANTS count", len(ALL_VARIANTS), 7)
check("constants/VARIANT_PRESETS keys", sorted(VARIANT_PRESETS.keys()), ["fast", "full"])
# --------------------------------------------------------------- make_variants
v1 = make_variants(["a", "b", "c"], "seed")
v2 = make_variants(["a", "b", "c"], "seed")
check("make_variants/deterministic", v1, v2)
check("make_variants/original is first", v1[0], ("original", ["a", "b", "c"], False))
check("make_variants/no duplicates", len({(tuple(o), r) for _, o, r in v1}), len(v1))
# 2 options: original + reversed + letters + letters_reversed = at most 4 variants
check_true("make_variants/two options capped at 4", len(make_variants(["x", "y"], "s", n_shuffles=10)) <= 4)
# Presets
fast = make_variants(["a", "b", "c"], "s", include=VARIANT_PRESETS["fast"])
check("make_variants/fast preset has 3 rows", [name for name, _, _ in fast], ["original", "reversed", "letters_reversed"])
# Relabel flag disabled
no_rel = make_variants(["a", "b", "c"], "s", relabel=False)
check_true("make_variants/no relabel has no letters variants", all(not r for _, _, r in no_rel))
# --------------------------------------------------------------- validation
check_raises("validation/missing required accept and escalate", TypeError, lambda: OptionStabilityHook())
check_raises("validation/negative n_shuffles", ValueError, lambda: OptionStabilityHook(accept=0.8, escalate=0.6, n_shuffles=-1))
check_raises("validation/bool n_shuffles", ValueError, lambda: OptionStabilityHook(accept=0.8, escalate=0.6, n_shuffles=True))
check_raises("validation/bool relabel", TypeError, lambda: OptionStabilityHook(accept=0.8, escalate=0.6, relabel="yes"))
check_raises("validation/unknown preset", ValueError, lambda: OptionStabilityHook(accept=0.8, escalate=0.6, variants="invalid"))
check_raises("validation/empty variants list", ValueError, lambda: OptionStabilityHook(accept=0.8, escalate=0.6, variants=[]))
check_raises("validation/unknown variant name", ValueError, lambda: OptionStabilityHook(accept=0.8, escalate=0.6, variants=["reversed", "nope"]))
check_raises("validation/invalid accept float", ValueError, lambda: OptionStabilityHook(accept=1.5, escalate=0.5))
check_raises("validation/negative accept", ValueError, lambda: OptionStabilityHook(accept=-0.1, escalate=0.0))
check_raises("validation/invalid escalate float", ValueError, lambda: OptionStabilityHook(accept=0.9, escalate=1.1))
check_raises("validation/escalate greater than accept", ValueError, lambda: OptionStabilityHook(accept=0.5, escalate=0.8))
# --------------------------------------------------------------- decide & calibrate
check("decide/accept", decide_reliability(0.95, accept=0.77, escalate=0.61), "ACCEPT")
check("decide/verify", decide_reliability(0.70, accept=0.77, escalate=0.61), "VERIFY")
check("decide/escalate", decide_reliability(0.30, accept=0.77, escalate=0.61), "ESCALATE")
check_raises("decide/missing accept", TypeError, lambda: decide_reliability(0.95))
check_raises("calibrate/empty pairs", ValueError, lambda: calibrate_reliability([], []))
scores = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 0.99]
correct = [False, False, False, True, False, True, False, True, True, True, True, True]
acc, esc = calibrate_reliability(scores, correct, target_accuracy=0.90)
check_true("calibrate/escalate <= accept", esc <= acc)
kept = [c for s, c in zip(scores, correct) if s >= acc]
check_true("calibrate/target accuracy reached", sum(kept) / len(kept) >= 0.90)
# Invalid evidence must not fit a usable gate by silently losing rows (#1312).
complete_cuts = calibrate_reliability([0.9, 0.9], [True, False])
check("calibrate/complete tied evidence accepts nothing", complete_cuts, (1.01, 0.0))
check("calibrate/complete tied evidence replays as VERIFY",
decide_reliability(0.9, *complete_cuts), "VERIFY")
for name, values, labels in (
("short labels", [0.9, 0.9], [True]),
("short scores", [0.9], [True, False]),
("empty labels", [0.9], []),
("empty scores", [], [True]),
("NaN first", [float("nan"), 0.9], [False, True]),
("NaN last", [0.9, float("nan")], [True, False]),
):
check_raises("calibrate/rejects " + name, ValueError,
lambda: calibrate_reliability(values, labels))
invalid_probabilities = (
-0.1, 1.01, float("nan"), float("inf"), -float("inf"),
True, False, "0.9", None, 0.9 + 0j, 10 ** 1000,
)
for value in invalid_probabilities:
check_raises("calibrate/rejects score %r" % (value,), ValueError,
lambda: calibrate_reliability([0.9, value], [True, False]))
for parameter in ("target_accuracy", "max_escalate_accuracy"):
check_raises("calibrate/rejects %s=%r" % (parameter, value), ValueError,
lambda: calibrate_reliability([0.9], [False], **{parameter: value}))
for label in (0, 1, 2, 0.0, 1.0, "false", None):
check_raises("calibrate/rejects non-boolean label %r" % (label,), TypeError,
lambda: calibrate_reliability([0.9, 0.9], [True, label]))
# ... but a NumPy boolean IS a boolean, and `correct` is normally computed rather than written
# out: `preds == gold` on a NumPy array or a pandas column yields `np.bool_`, which fails a bare
# `isinstance(x, bool)`. Same for `np.float32`/`np.int64` scores, which `numbers.Real` accepts
# while still excluding `np.bool_`, so the score and label checks stay disjoint. The fitted
# cutoffs come back as plain floats either way, because a caller stores and replays them.
import numpy as _np # noqa: E402
_want = calibrate_reliability([0.9, 0.4, 0.8], [True, False, True], target_accuracy=1.0)
check("calibrate/plain floats and bools", _want, (0.8, 0.0))
for _name, _scores, _labels in (
("numpy bool array labels", [0.9, 0.4, 0.8], _np.array([True, False, True])),
("numpy bool scalars", [0.9, 0.4, 0.8], list(_np.array([1, 0, 1]) == 1)),
("numpy float32 scores", list(_np.array([0.9, 0.4, 0.8], dtype=_np.float32)), [True, False, True]),
("numpy float64 scores", list(_np.array([0.9, 0.4, 0.8])), [True, False, True]),
("numpy int scores", list(_np.array([1, 0, 1], dtype=_np.int64)), [True, False, True]),
):
_got = calibrate_reliability(_scores, _labels, target_accuracy=1.0)
check_true("calibrate/%s fits" % _name, _got[0] <= 1.0 and _got[1] == 0.0, repr(_got))
check("calibrate/%s returns plain floats" % _name,
[type(_c).__name__ for _c in _got], ["float", "float"])
check("calibrate/numpy bools fit the same cutoffs as python bools",
calibrate_reliability([0.9, 0.4, 0.8], _np.array([True, False, True]), target_accuracy=1.0), _want)
# A NumPy boolean is still not a score, and a NumPy float is still not a label.
check_raises("calibrate/rejects numpy bool score", ValueError,
lambda: calibrate_reliability([_np.True_], [True]))
check_raises("calibrate/rejects numpy float label", TypeError,
lambda: calibrate_reliability([0.9], [_np.float64(1.0)]))
# Sized sequences only: a length mismatch has to be detectable, so a generator is refused.
check_raises("calibrate/rejects a generator of scores", TypeError,
lambda: calibrate_reliability((x for x in [0.9]), [True]))
# Integer endpoints and tuple sequences remain valid; fitting must not mutate callers.
for target in (0, 1):
for max_escalate in (0, 1):
cuts = calibrate_reliability((0, 1), (False, True), target, max_escalate)
check("calibrate/target=%d max_escalate=%d endpoints" % (target, max_escalate),
cuts, (target, 0.0))
check("calibrate/target=%d max_escalate=%d replay" % (target, max_escalate),
[decide_reliability(score, *cuts) for score in (0, 1)],
["ACCEPT", "ACCEPT"] if target == 0 else ["VERIFY", "ACCEPT"])
unsorted_scores, unsorted_labels = [0.9, 0.1, 0.5], [True, False, False]
check("calibrate/unsorted evidence", calibrate_reliability(unsorted_scores, unsorted_labels), (0.9, 0.5))
check("calibrate/does not mutate evidence", (unsorted_scores, unsorted_labels),
([0.9, 0.1, 0.5], [True, False, False]))
# --------------------------------------------------------------- hook lifecycle: single state
hook = OptionStabilityHook(accept=0.77, escalate=0.61, seed="fixed-seed")
q_choice = {
"ask": {
"type": "choice",
"instructions": "Which team?",
"criteria": {"refund": "money back", "tech": "bug fix", "sales": "pricing"},
},
"score_q": {"type": "score", "instructions": "Rate?", "criteria": ["low", "high"]},
"noul_q": {"type": "noul", "instructions": "Yes or no?"},
}
ctx = PredictContext(states=["I want my money back."], questions=dict(q_choice))
hook.on_predict_start(ctx)
# Check variants added
check_true("start/original questions preserved", "ask" in ctx.questions)
check_true("start/non-choice unchanged", "score_q" in ctx.questions and "noul_q" in ctx.questions)
variant_keys = [k for k in ctx.questions if k.startswith("ask::rel")]
check_true("start/variants added for choice", len(variant_keys) >= 2)
# Check reserved separator rejected in caller question ids
bad_ctx = PredictContext(states=["test"], questions={"bad::rel1": q_choice["ask"]})
check_raises("start/rejects reserved separator", ValueError, lambda: hook.on_predict_start(bad_ctx))
# Simulate model execution where "refund" wins across all variants (stable agent)
simulated_answers = {}
for qk, qv in ctx.questions.items():
if qk == "score_q":
simulated_answers[qk] = {"type": "score", "score": 1.0, "confidence": 0.9, "answer_confidence": 0.9}
elif qk == "noul_q":
simulated_answers[qk] = {"type": "noul", "noul": 0.8, "confidence": 0.8, "answer_confidence": 0.8}
else:
# Choice question (original or variant)
crit = qv["criteria"]
# Find which key holds description "money back"
target_key = [k for k, desc in crit.items() if desc == "money back"][0]
other_keys = [k for k in crit if k != target_key]
probs = {target_key: 0.85}
for ok in other_keys:
probs[ok] = round(0.15 / len(other_keys), 4)
simulated_answers[qk] = {
"type": "choice",
"choice": target_key,
"probabilities": probs,
"confidence": 0.7,
"answer_confidence": 0.85,
}
ctx.results = [{
"model": "fake",
"answers": simulated_answers,
"usage": {
"input_tokens": 100,
"output_tokens": 0,
"truncated_questions": ["ask::rel1"],
"options": {"ask": {"total": 3, "distinct": 3}, "ask::rel1": {"total": 3, "distinct": 3}},
},
}]
hook.on_predict_end(ctx)
# Verify results after on_predict_end
res = ctx.results[0]
ans = res["answers"]
check_true("end/variant keys removed from answers", not any(k.startswith("ask::rel") for k in ans))
check("end/original choice question retained", ans["ask"]["choice"], "refund")
check_true("end/reliability added to choice", "reliability" in ans["ask"])
rel = ans["ask"]["reliability"]
check("end/stable answer accepted", rel["decision"], "ACCEPT")
check("end/stability is 1.0", rel["stability"], 1.0)
check_true("end/soft_stability is ~0.85", abs(rel["soft_stability"] - 0.85) < 1e-3)
check("end/distinct_choices is single refund", rel["distinct_choices"], ["refund"])
check_true("end/non-choice has no reliability", "reliability" not in ans["score_q"])
check_true("end/usage truncated_questions cleaned", "ask::rel1" not in res["usage"]["truncated_questions"])
check_true("end/usage options cleaned", "ask::rel1" not in res["usage"]["options"])
check_true("end/internal plan cleaned", ctx.run_id not in hook._plans)
# --------------------------------------------------------------- fragile agent (picks first option)
ctx_fragile = PredictContext(states=["test"], questions={"ask": dict(q_choice["ask"])})
hook_fragile = OptionStabilityHook(accept=0.77, escalate=0.61, seed="fixed-seed")
hook_fragile.on_predict_start(ctx_fragile)
sim_fragile = {}
for qk, qv in ctx_fragile.questions.items():
crit = qv["criteria"]
keys = list(crit.keys())
probs = {keys[0]: 0.9}
for ok in keys[1:]:
probs[ok] = round(0.1 / (len(keys) - 1), 4)
sim_fragile[qk] = {
"type": "choice",
"choice": keys[0],
"probabilities": probs,
"confidence": 0.8,
"answer_confidence": 0.9,
}
ctx_fragile.results = [{"model": "fake", "answers": sim_fragile, "usage": {}}]
hook_fragile.on_predict_end(ctx_fragile)
rel_f = ctx_fragile.results[0]["answers"]["ask"]["reliability"]
check_true("fragile/stability is low", rel_f["stability"] < 0.5)
check_true("fragile/decision is not ACCEPT", rel_f["decision"] in ("VERIFY", "ESCALATE"))
check_true("fragile/multiple distinct choices", len(rel_f["distinct_choices"]) > 1)
# --------------------------------------------------------------- batch execution
ctx_batch = PredictContext(states=["s1", "s2"], questions={"ask": dict(q_choice["ask"])})
hook_batch = OptionStabilityHook(accept=0.77, escalate=0.61, seed="batch-test")
hook_batch.on_predict_start(ctx_batch)
# Simulate 2 states in results
results_batch = []
for state in ("s1", "s2"):
sim_b = {}
for qk, qv in ctx_batch.questions.items():
crit = qv["criteria"]
target = [k for k, desc in crit.items() if desc == "money back"][0]
probs = {k: (0.9 if k == target else 0.05) for k in crit}
sim_b[qk] = {"type": "choice", "choice": target, "probabilities": probs, "answer_confidence": 0.9}
results_batch.append({"model": "fake", "answers": sim_b, "usage": {}})
ctx_batch.results = results_batch
hook_batch.on_predict_end(ctx_batch)
check("batch/len results is 2", len(ctx_batch.results), 2)
for i, r in enumerate(ctx_batch.results):
a = r["answers"]["ask"]
check("batch/%d variant keys cleaned" % i, [k for k in r["answers"] if "::rel" in k], [])
check("batch/%d decision" % i, a["reliability"]["decision"], "ACCEPT")
# --------------------------------------------------------------- error handling cleanup
ctx_err = PredictContext(states=["err"], questions={"ask": dict(q_choice["ask"])})
hook_err = OptionStabilityHook(accept=0.77, escalate=0.61)
hook_err.on_predict_start(ctx_err)
check_true("error/plan exists before error", ctx_err.run_id in hook_err._plans)
hook_err.on_error(ctx_err)
check_true("error/plan cleared on error", ctx_err.run_id not in hook_err._plans)
# --------------------------------------------------------------- uncalibrated / accept=None
check("decide/uncalibrated returns None", decide_reliability(0.99, accept=None), None)
hook_uncal = OptionStabilityHook(accept=None, escalate=None, seed="fixed-seed")
check("hook/uncalibrated accept is None", hook_uncal.accept, None)
check("hook/uncalibrated escalate is None", hook_uncal.escalate, None)
ctx_uncal = PredictContext(states=["I want my money back."], questions={"ask": dict(q_choice["ask"])})
hook_uncal.on_predict_start(ctx_uncal)
sim_uncal = {}
for qk, qv in ctx_uncal.questions.items():
crit = qv["criteria"]
target_key = [k for k, desc in crit.items() if desc == "money back"][0]
sim_uncal[qk] = {
"type": "choice",
"choice": target_key,
"probabilities": {target_key: 0.85},
"confidence": 0.85,
"answer_confidence": 0.85,
}
ctx_uncal.results = [{"answers": sim_uncal, "usage": {}}]
hook_uncal.on_predict_end(ctx_uncal)
check("hook/uncalibrated decision is None", ctx_uncal.results[0]["answers"]["ask"]["reliability"]["decision"], None)
# --------------------------------------------------------------- layout-aware probe selection (#635, #951)
class MockAgent:
def __init__(self, option_layout="sequential"):
self.config = {"option_layout": option_layout}
# 1. Parallel layout: auto-probes run rename-only
parallel_agent = MockAgent(option_layout="parallel")
ctx_par = PredictContext(states=["test parallel"], questions={"ask": dict(q_choice["ask"])}, agent=parallel_agent)
hook_par = OptionStabilityHook(accept=0.77, escalate=0.61, seed="fixed-seed")
hook_par.on_predict_start(ctx_par)
# Only rename variants should be injected (no reversed or shuffle)
par_variant_names = [k for k in ctx_par.questions.keys() if k != "ask"]
check_true("parallel/injects variants", len(par_variant_names) > 0)
check("parallel/plan probes_ran is rename only", hook_par._plans[ctx_par.run_id]["ask"]["probes_ran"], ["rename"])
check("parallel/plan layout is parallel", hook_par._plans[ctx_par.run_id]["ask"]["layout"], "parallel")
sim_par = {}
for qk, qv in ctx_par.questions.items():
crit = qv["criteria"]
target_key = [k for k, desc in crit.items() if desc == "money back"][0]
sim_par[qk] = {
"type": "choice",
"choice": target_key,
"probabilities": {target_key: 0.90},
"confidence": 0.90,
"answer_confidence": 0.90,
}
ctx_par.results = [{"answers": sim_par, "usage": {}}]
hook_par.on_predict_end(ctx_par)
par_rel = ctx_par.results[0]["answers"]["ask"]["reliability"]
check("parallel/reliability probes is ['rename']", par_rel["probes"], ["rename"])
check("parallel/reliability layout is parallel", par_rel["option_layout"], "parallel")
check_true("parallel/rename_stability is not None", par_rel["rename_stability"] is not None)
check("parallel/reorder_stability is None", par_rel["reorder_stability"], None)
# 2. Sequential layout: auto-probes run both reorder and rename
seq_agent = MockAgent(option_layout="sequential")
ctx_seq = PredictContext(states=["test sequential"], questions={"ask": dict(q_choice["ask"])}, agent=seq_agent)
hook_seq = OptionStabilityHook(accept=0.77, escalate=0.61, seed="fixed-seed")
hook_seq.on_predict_start(ctx_seq)
check("sequential/plan probes_ran is ['reorder', 'rename']", hook_seq._plans[ctx_seq.run_id]["ask"]["probes_ran"], ["reorder", "rename"])
sim_seq = {}
for qk, qv in ctx_seq.questions.items():
crit = qv["criteria"]
target_key = [k for k, desc in crit.items() if desc == "money back"][0]
sim_seq[qk] = {
"type": "choice",
"choice": target_key,
"probabilities": {target_key: 0.90},
"confidence": 0.90,
"answer_confidence": 0.90,
}
ctx_seq.results = [{"answers": sim_seq, "usage": {}}]
hook_seq.on_predict_end(ctx_seq)
seq_rel = ctx_seq.results[0]["answers"]["ask"]["reliability"]
check("sequential/reliability probes is ['reorder', 'rename']", seq_rel["probes"], ["reorder", "rename"])
check_true("sequential/rename_stability is not None", seq_rel["rename_stability"] is not None)
check_true("sequential/reorder_stability is not None", seq_rel["reorder_stability"] is not None)
# 3. Explicit probe override: probes="reorder" on parallel checkpoint overrides auto
ctx_forced = PredictContext(states=["test forced"], questions={"ask": dict(q_choice["ask"])}, agent=parallel_agent)
hook_forced = OptionStabilityHook(accept=0.77, escalate=0.61, probes="reorder", seed="fixed-seed")
hook_forced.on_predict_start(ctx_forced)
check("override/plan probes_ran is reorder", hook_forced._plans[ctx_forced.run_id]["ask"]["probes_ran"], ["reorder"])
# 4. Unprobed single-option question: returns None signals and probes=[]
single_q = {"single": {"type": "choice", "instructions": "only one", "criteria": {"a": "only"}}}
ctx_single = PredictContext(states=["test single"], questions=single_q, agent=seq_agent)
hook_single = OptionStabilityHook(accept=0.77, escalate=0.61)
hook_single.on_predict_start(ctx_single)
check("single_option/no variants injected", len(ctx_single.questions), 1)
ctx_single.results = [{"answers": {"single": {"type": "choice", "choice": "a", "probabilities": {"a": 1.0}}}, "usage": {}}]
hook_single.on_predict_end(ctx_single)
single_rel = ctx_single.results[0]["answers"]["single"]["reliability"]
check("single_option/probes is empty", single_rel["probes"], [])
check("single_option/stability is None", single_rel["stability"], None)
check("single_option/soft_stability is None", single_rel["soft_stability"], None)
check("single_option/decision is None", single_rel["decision"], None)
# 5. Validation of new parameters
check_raises("validation/invalid probe string", ValueError, lambda: OptionStabilityHook(accept=0.8, escalate=0.6, probes="nonsense"))
check_raises("validation/empty probe list", ValueError, lambda: OptionStabilityHook(accept=0.8, escalate=0.6, probes=[]))
check_raises("validation/invalid probe list item", ValueError, lambda: OptionStabilityHook(accept=0.8, escalate=0.6, probes=["reorder", "bad"]))
check_raises("validation/invalid rename_max_options", ValueError, lambda: OptionStabilityHook(accept=0.8, escalate=0.6, rename_max_options=1))
check_raises("validation/bool rename_max_options", ValueError, lambda: OptionStabilityHook(accept=0.8, escalate=0.6, rename_max_options=True))
# 6. Layout normalization and warning fallback
title_par_agent = MockAgent(option_layout=" Parallel ")
ctx_title_par = PredictContext(states=["test case normalization"], questions={"ask": dict(q_choice["ask"])}, agent=title_par_agent)
hook_norm = OptionStabilityHook(accept=0.77, escalate=0.61, seed="fixed-seed")
hook_norm.on_predict_start(ctx_title_par)
check("normalization/Parallel maps to parallel layout", hook_norm._plans[ctx_title_par.run_id]["ask"]["layout"], "parallel")
import warnings as _test_warnings
with _test_warnings.catch_warnings(record=True) as _warn_records:
_test_warnings.simplefilter("always")
bogus_agent = MockAgent(option_layout="unsupported_layout")
ctx_bogus = PredictContext(states=["test bogus layout"], questions={"ask": dict(q_choice["ask"])}, agent=bogus_agent)
hook_bogus = OptionStabilityHook(accept=0.77, escalate=0.61, seed="fixed-seed")
hook_bogus.on_predict_start(ctx_bogus)
check("normalization/bogus falls back to sequential", hook_bogus._plans[ctx_bogus.run_id]["ask"]["layout"], "sequential")
check_true("normalization/bogus emits warning", any("unknown option_layout" in str(w.message) for w in _warn_records))
# 7. Issue #1285: Preserve option labels when renaming choices without descriptions
q_dict_none = {"type": "choice", "instructions": "Action?", "criteria": {"approve": None, "reject": None}}
vq_none_rel, back_none_rel = OptionStabilityHook._variant_question(q_dict_none, ["approve", "reject"], relabel=True)
check("issue1285/none_desc relabel preserves labels as descriptions", vq_none_rel["criteria"], {"A": "approve", "B": "reject"})
check("issue1285/none_desc back map", back_none_rel, {"A": "approve", "B": "reject"})
vq_none_ord, back_none_ord = OptionStabilityHook._variant_question(q_dict_none, ["reject", "approve"], relabel=False)
check("issue1285/none_desc reorder preserves original keys and values", vq_none_ord["criteria"], {"reject": None, "approve": None})
check("issue1285/none_desc reorder back map", back_none_ord, {"reject": "reject", "approve": "approve"})
q_dict_empty = {"type": "choice", "instructions": "Action?", "criteria": {"approve": "", "reject": ""}}
vq_empty_rel, _ = OptionStabilityHook._variant_question(q_dict_empty, ["approve", "reject"], relabel=True)
check("issue1285/empty_str relabel preserves labels as descriptions", vq_empty_rel["criteria"], {"A": "approve", "B": "reject"})
q_dict_ws = {"type": "choice", "instructions": "Action?", "criteria": {"approve": " ", "reject": " "}}
vq_ws_rel, _ = OptionStabilityHook._variant_question(q_dict_ws, ["approve", "reject"], relabel=True)
check("issue1285/whitespace relabel preserves labels as descriptions", vq_ws_rel["criteria"], {"A": "approve", "B": "reject"})
q_list = {"type": "choice", "instructions": "Action?", "criteria": ["approve", "reject"]}
vq_list_rel, _ = OptionStabilityHook._variant_question(q_list, ["approve", "reject"], relabel=True)
check("issue1285/list and dict forms produce identical rename criteria", vq_none_rel["criteria"], vq_list_rel["criteria"])
q_mixed = {"type": "choice", "instructions": "Action?", "criteria": {"approve": "Confirm order", "reject": None}}
vq_mixed_rel, _ = OptionStabilityHook._variant_question(q_mixed, ["approve", "reject"], relabel=True)
check("issue1285/mixed keeps text description and fills missing with label", vq_mixed_rel["criteria"], {"A": "Confirm order", "B": "reject"})
q_falsy = {"type": "choice", "instructions": "Action?", "criteria": {"level_0": 0, "flag": False}}
vq_falsy_rel, _ = OptionStabilityHook._variant_question(q_falsy, ["level_0", "flag"], relabel=True)
check("issue1285/falsy numeric and boolean descriptions preserved", vq_falsy_rel["criteria"], {"A": 0, "B": False})
# End-to-end hook execution with undescribed dict criteria (#1285)
ctx_undescribed = PredictContext(
states=["Please confirm my invoice"],
questions={"action": {"type": "choice", "instructions": "Choose action", "criteria": {"approve": None, "reject": None}}},
)
hook_undescribed = OptionStabilityHook(accept=0.77, escalate=0.61, seed="undescribed-test")
hook_undescribed.on_predict_start(ctx_undescribed)
# Original question criteria must remain untouched
check("issue1285/e2e original question criteria unchanged", ctx_undescribed.questions["action"]["criteria"], {"approve": None, "reject": None})
# Injected rename variant questions have original labels as descriptions
rename_variants = [q for k, q in ctx_undescribed.questions.items() if k.startswith("action::rel") and list(q["criteria"].keys())[0] == "A"]
check_true("issue1285/e2e rename variants present", len(rename_variants) > 0)
for rv in rename_variants:
check("issue1285/e2e rename variant preserves option semantic text", set(rv["criteria"].values()), {"approve", "reject"})
# Simulate inference response and verify back-mapping restores original labels
sim_answers = {}
for qk, qv in ctx_undescribed.questions.items():
crit = qv["criteria"]
# Model chooses option representing "approve"
chosen_key = [k for k, v in crit.items() if (v == "approve" or k == "approve")][0]
other_key = [k for k in crit if k != chosen_key][0]
sim_answers[qk] = {
"type": "choice",
"choice": chosen_key,
"probabilities": {chosen_key: 0.9, other_key: 0.1},
"confidence": 0.8,
"answer_confidence": 0.9,
}
ctx_undescribed.results = [{"model": "fake", "answers": sim_answers, "usage": {}}]
hook_undescribed.on_predict_end(ctx_undescribed)
ans_e2e = ctx_undescribed.results[0]["answers"]["action"]
check("issue1285/e2e decoded choice is approve", ans_e2e["choice"], "approve")
check("issue1285/e2e distinct choices is approve", ans_e2e["reliability"]["distinct_choices"], ["approve"])
check("issue1285/e2e stability is 1.0", ans_e2e["reliability"]["stability"], 1.0)
check("issue1285/e2e decision is ACCEPT", ans_e2e["reliability"]["decision"], "ACCEPT")
# Parity check: list criteria vs dict-with-None criteria generate matching variants
ctx_list_parity = PredictContext(
states=["Please confirm my invoice"],
questions={"action": {"type": "choice", "instructions": "Choose action", "criteria": ["approve", "reject"]}},
)
hook_list_parity = OptionStabilityHook(accept=0.77, escalate=0.61, seed="undescribed-test")
hook_list_parity.on_predict_start(ctx_list_parity)
check("issue1285/parity injected variant count matches", len(ctx_undescribed.questions), len(ctx_list_parity.questions))
for (k1, q1), (k2, q2) in zip(sorted(ctx_undescribed.questions.items()), sorted(ctx_list_parity.questions.items())):
if k1.startswith("action::rel"):
c1, c2 = q1["criteria"], q2["criteria"]
if isinstance(c1, dict) and isinstance(c2, dict):
check("issue1285/parity rename criteria match %s" % k1, c1, c2)
else:
# Reorder probe: dict keys match list entries exactly
check("issue1285/parity reorder ordering matches %s" % k1, list(c1.keys()), list(c2))
hook_list_parity.on_error(ctx_list_parity)
# --------------------------------------------------------------- calibration precision and replay
def precision_report(supports, accept=None, escalate=None):
"""Use real hook aggregation over simulated per-variant model probabilities."""
precision_hook = OptionStabilityHook(accept=accept, escalate=escalate, variants="fast")
precision_ctx = PredictContext(states=["Keep the request"], questions={
"action": {"type": "choice", "instructions": "Which action?",
"criteria": {"keep": "keep request", "defer": "defer request"}},
})
precision_hook.on_predict_start(precision_ctx)
assert len(precision_ctx.questions) == len(supports) == 3
answers = {}
for (qid, question), support in zip(precision_ctx.questions.items(), supports):
target = next(k for k, value in question["criteria"].items() if value == "keep request")
other = next(k for k in question["criteria"] if k != target)
answers[qid] = {"type": "choice", "choice": target if support >= 0.5 else other,
"probabilities": {target: support, other: 1.0 - support}}
precision_ctx.results = [{"answers": answers, "usage": {}}]
precision_hook.on_predict_end(precision_ctx)
return precision_ctx.results[0]["answers"]["action"]["reliability"]
# The individual probabilities have four decimal places; their mean generally does not.
# A saved report must reproduce the live decision without changing caller-supplied cuts.
for name, supports, accept, escalate, expected in (
("accept round-up", [0.7999, 0.8, 0.8], 0.8, 0.6, "VERIFY"),
("escalate round-up", [0.5999, 0.6, 0.6], 0.8, 0.6, "ESCALATE"),
("precise caller cut", [0.8001, 0.8, 0.8], 0.80002, 0.6, "ACCEPT"),
("uncalibrated", [0.7999, 0.8, 0.8], None, None, None),
):
report = precision_report(supports, accept, escalate)
restored = json.loads(json.dumps(report))
check("precision/%s live decision" % name, report["decision"], expected)
check("precision/%s reports the gated mean" % name, report["soft_stability"], sum(supports) / 3)
check("precision/%s JSON replay" % name,
decide_reliability(restored["soft_stability"], accept, escalate), expected)
# Fit from persisted reports, then replay the exact same model outputs. The lower two
# scores round to the same four-place value, but each is a distinct possible cutoff.
calibration_supports = [
[0.8, 0.6, 0.7],
[0.8, 0.6, 0.7001],
[0.8, 0.6, 0.7002],
]
saved_reports = json.loads(json.dumps([precision_report(row) for row in calibration_supports]))
saved_scores = [report["soft_stability"] for report in saved_reports]
fitted = calibrate_reliability(saved_scores, [False, False, True],
target_accuracy=1.0, max_escalate_accuracy=0.0)
fitted = json.loads(json.dumps(fitted))
check("precision/fitted accept is the observed upper mean", fitted[0], sum(calibration_supports[2]) / 3)
check("precision/fitted escalate is the observed middle mean", fitted[1], sum(calibration_supports[1]) / 3)
check("precision/JSON fit and hook replay preserve all three bands",
[precision_report(row, *fitted)["decision"] for row in calibration_supports],
["ESCALATE", "VERIFY", "ACCEPT"])
check("precision/escalate boundary is exclusive",
decide_reliability(fitted[1], *fitted), "VERIFY")
check("precision/accept boundary is inclusive",
decide_reliability(fitted[0], *fitted), "ACCEPT")
# Rounding a sole candidate up can leave no retained row; rounding nearby candidates
# down can merge a wrong row into a group that has a valid perfect-accuracy cutoff.
for name, values, labels, expected_cut in (
("round-up retains a lone correct row", [0.80006], [True], 0.80006),
("round-down keeps nearby scores separable", [0.80001, 0.80002], [False, True], 0.80002),
):
accept, escalate = calibrate_reliability(values, labels, target_accuracy=1.0)
check("precision/calibrate %s" % name, accept, expected_cut)
accepted = [ok for value, ok in zip(values, labels)
if decide_reliability(value, accept, escalate) == "ACCEPT"]
check("precision/calibrate %s retained group" % name, accepted, [True])
check("precision/exact ties cannot be split",
calibrate_reliability([0.8, 0.8], [False, True], target_accuracy=1.0), (1.01, 0.0))
check("precision/duplicate scores enter as a whole group",
calibrate_reliability([0.8, 0.8], [False, True], target_accuracy=0.5), (0.8, 0.0))
check("precision/zero and one retain their boundary cuts",
calibrate_reliability([0.0, 1.0], [False, True], target_accuracy=1.0), (1.0, 0.0))
check("precision/zero is accepted at a zero cutoff", decide_reliability(0.0, 0.0, 0.0), "ACCEPT")
check("precision/sentinel accepts no probability", decide_reliability(1.0, 1.01, 0.0), "VERIFY")
# --------------------------------------------------------------- report
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL", f)
if not FAIL:
print("all option stability tests passed")
sys.exit(1 if FAIL else 0)