Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

142 KB
Raw Blame
"""`confidence` carries two definitions, and only one of them is the calibrated quantity.

    choice / score  ->  confidence_from_probs(p, k) == 1 - H(p) / log(k)   (k >= 2)
    noul            ->  max(p_true, 1 - p_true)     == max(p)

They are not on the same scale. The same two-option distribution comes back as 0.90 from a
`noul` and 0.53 from an equivalent two-option `choice`, and every shipped preset mixes the two
types in one call -- `moderation_questions()` is four `noul` and one `score`. The README's
"Automated Confidence Gating" section gates all of them on a single threshold.

Both benchmark harnesses in research/scripts take `conf = max(probs)` before calling
`ece_score`, so every ECE figure in the README describes max(p) and not normalized entropy.
`answer_confidence` reports that quantity on every question type, additively: `confidence` is
untouched, so nothing a caller gates on today moves.

No weights are loaded: the confidence helpers are pure.
"""
import copy
import math
import os
import sys

import numpy as np

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from laya.common import answer_confidence, confidence_from_probs, ece_score  # noqa: E402

PASS, FAIL = [], []


def check(name, got, want):
    if got == want:
        PASS.append(name)
    else:
        FAIL.append("%s: got %r, want %r" % (name, got, want))


def check_true(name, cond, detail=""):
    if cond:
        PASS.append(name)
    else:
        FAIL.append("%s %s" % (name, detail))


def close(a, b, tol=1e-9):
    return abs(a - b) <= tol


# --------------------------------------------------------------- answer_confidence is max(p)
for probs in ([0.5, 0.5], [0.1, 0.9], [0.7, 0.2, 0.1], [0.25] * 4, [1.0, 0.0, 0.0]):
    p = np.array(probs)
    check_true("answer/max of %s" % (probs,), close(answer_confidence(p, len(probs)), max(probs)))

check("answer/only the first k entries count", answer_confidence(np.array([0.4, 0.6, 0.99]), 2), 0.6)
check("answer/k=1 is certain", answer_confidence(np.array([1.0]), 1), 1.0)
check("answer/k=0 does not crash", answer_confidence(np.array([]), 0), 1.0)

# --------------------------------------------------------------- it matches noul's definition
# `noul` reports max(p_true, 1 - p_true), which over two options is exactly max(p). So for a
# noul answer the new field equals the existing one, and the two agree by construction.
for p_true in (0.0, 0.05, 0.3, 0.5, 0.62, 0.9, 1.0):
    p = np.array([1.0 - p_true, p_true])
    check_true("noul/answer_confidence equals the shipped noul confidence at p=%.2f" % p_true,
               close(answer_confidence(p, 2), max(p_true, 1.0 - p_true)))

# --------------------------------------------------------------- the two scales really differ
# Same distribution, two question types. 0.85 is the threshold the README's gating section uses.
disagree = []
for p_true in (0.60, 0.70, 0.80, 0.85, 0.90, 0.95):
    p = np.array([1.0 - p_true, p_true])
    if (answer_confidence(p, 2) >= 0.85) != (confidence_from_probs(p, 2) >= 0.85):
        disagree.append(p_true)
check_true("scales/the two disagree across the documented threshold", disagree == [0.85, 0.90, 0.95],
           "disagreed at %s" % (disagree,))
check_true("scales/entropy sits far below max(p) on the same distribution",
           confidence_from_probs(np.array([0.1, 0.9]), 2) < 0.55 < answer_confidence(np.array([0.1, 0.9]), 2))
# max(p) over two options has a floor of 0.5; entropy reads 0.0 for the same coin flip.
check("scales/a coin flip is 0.5 on max(p)", answer_confidence(np.array([0.5, 0.5]), 2), 0.5)
check("scales/and 0.0 on entropy", confidence_from_probs(np.array([0.5, 0.5]), 2), 0.0)

# --------------------------------------------------------------- only one of them is calibrated
# Perfectly calibrated predictions: an answer reported at top probability c is right exactly c
# of the time. ECE on max(p) must be near zero. ECE on normalized entropy must not be, which is
# why it cannot be compared against a probability threshold.
rng = np.random.default_rng(0)
tops, ents, correct = [], [], []
for c in np.linspace(0.30, 0.99, 24):
    rest = (1.0 - c) / 2.0
    p = np.array([c, rest, rest])
    for _ in range(400):
        tops.append(answer_confidence(p, 3))
        ents.append(confidence_from_probs(p, 3))
        correct.append(1.0 if rng.random() < c else 0.0)

tops, ents, correct = np.array(tops), np.array(ents), np.array(correct)
ece_top, ece_ent = ece_score(tops, correct), ece_score(ents, correct)
check_true("calibration/max(p) is calibrated on calibrated data (ECE %.4f)" % ece_top,
           ece_top < 0.03, "ECE %.4f" % ece_top)
check_true("calibration/entropy is not (ECE %.4f)" % ece_ent, ece_ent > 0.20, "ECE %.4f" % ece_ent)
check_true("calibration/entropy is worse by a wide margin", ece_ent > 5 * ece_top,
           "top %.4f vs entropy %.4f" % (ece_top, ece_ent))

# --------------------------------------------------------------- the entropy helper is untouched
for probs, k in (([0.1, 0.9], 2), ([0.25] * 4, 4), ([0.7, 0.2, 0.1], 3)):
    p = np.array(probs)
    ent = -(p * np.log(np.clip(p, 1e-12, 1.0))).sum()
    check_true("entropy/formula for k=%d" % k,
               close(confidence_from_probs(p, k), float(np.clip(1 - ent / math.log(k), 0.0, 1.0))))

# The `k < 2` floor: `log(1)` and `log(0)` are zero, so the formula above has no scale there and
# the helper hands back its maximum instead. Nothing checked that. `laya.structured` does build a
# one-option `choice`, from `{"enum": ["solo"]}` -- only the upper bound (`MAX_OPTIONS`) is
# enforced -- so for such a field the reported `confidence` is this constant whichever way `p`
# points. The abstention gate is not that reader: `laya.confidence` compares
# `answer_confidence` first and falls back to `confidence` only when that field is absent.
check("entropy/k=1 is the maximum, not 0/0", confidence_from_probs(np.array([1.0]), 1), 1.0)
check("entropy/k=1 ignores the value", confidence_from_probs(np.array([0.0]), 1), 1.0)
check("entropy/k=0 is the maximum", confidence_from_probs(np.array([]), 0), 1.0)
check_true("entropy/k=1 is at or above every threshold value the validator accepts",
           all(confidence_from_probs(np.array([0.4]), 1) >= t for t in (0.0, 0.5, 1.0)))

# --------------------------------------------------------------- min_confidence validation & abstention (#361)
from laya.confidence import check_min_confidence, flag_low_confidence  # noqa: E402

# check_min_confidence validates [0.0, 1.0] and rejects non-floats and bools
for valid in (0.0, 0.5, 1.0, 0, 1, 0.85):
    check("min_confidence/valid %.2f" % valid, check_min_confidence(valid), float(valid))

for invalid in (True, False, -0.01, 1.01, -1.0, 2.0, float("nan"), float("inf"), float("-inf"), "0.5", None, [0.5],
                # A JSON integer wider than a double is the third way a client's threshold is not a
                # number: `math.isfinite` raises `OverflowError` rather than answering "not finite",
                # so this walked out of `check_min_confidence` -- and out of `validate_min_confidence`,
                # whose docstring says it "only wraps its ValueError as a clean invalid_min_confidence
                # the client can read" -- as an interpreter error instead of the refusal.
                10 ** 400, -(10 ** 400)):
    try:
        check_min_confidence(invalid)
        FAIL.append("min_confidence/should reject %r" % (invalid,))
    except ValueError:
        PASS.append("min_confidence/rejected %r" % (invalid,))

# flag_low_confidence marks answers where answer_confidence < threshold
sample_res = [{
    "answers": {
        "q_high": {"type": "choice", "choice": "a", "answer_confidence": 0.92, "confidence": 0.8},
        "q_low": {"type": "score", "score": 1, "answer_confidence": 0.45, "confidence": 0.4},
        "q_fallback": {"type": "choice", "choice": "b", "confidence": 0.3},
        "q_exact": {"type": "choice", "choice": "c", "answer_confidence": 0.70},
    }
}]

# min_confidence=0.0 is a no-op (default behavior untouched)
flag_low_confidence(sample_res, 0.0)
check_true("flag/0.0 is no-op", not any("low_confidence" in a for a in sample_res[0]["answers"].values()))

# min_confidence=0.70: q_low (0.45) and q_fallback (0.3) flagged; q_high (0.92) and q_exact (0.70) NOT flagged
flag_low_confidence(sample_res, 0.70)
ans = sample_res[0]["answers"]
check_true("flag/q_low flagged", ans["q_low"].get("low_confidence") is True)
check_true("flag/q_fallback flagged", ans["q_fallback"].get("low_confidence") is True)
check_true("flag/q_high unflagged", "low_confidence" not in ans["q_high"])
check_true("flag/q_exact unflagged", "low_confidence" not in ans["q_exact"])
check("flag/answer_confidence intact", ans["q_low"]["answer_confidence"], 0.45)
check("flag/raw choice intact", ans["q_high"]["choice"], "a")

# --------------------------------------------------------------- the gate's state is reported
# `low_confidence` is written only when the gate fires, so its absence cannot tell a caller
# "a gate ran and this answer cleared it" from "no gate ran at all". An operator with 10,000
# logged decisions cannot compute an abstention rate, cannot tell whether a run was gated, and
# cannot re-split a batch that used different thresholds per request class -- the threshold is
# consumed and dropped. `docs/staged-adoption.md` asks a shadow record to carry "errors and any
# fallback or review decision"; there was no field to put that in.
#
# The contract: where a gate ran, its state is an explicit value, so it is never inferred from a
# missing field. Where no gate was configured, nothing is written at all -- an ungated call returns
# the payload it always returned, rather than growing a field every existing caller would then have
# to read. The two cases are told apart by the presence of `abstention`, which is why the vocabulary
# has no "not configured" member to read out of the field.
from laya.confidence import (  # noqa: E402
    GATE_ABSTAINED,
    GATE_PASSED,
    GATE_STATES,
    GATE_UNEVALUATED,
    apply_confidence_gate,
)

check("gate/the vocabulary is closed",
      list(GATE_STATES), ["passed", "abstained", "unevaluated"])

fresh = lambda: [{  # noqa: E731
    "answers": {
        "q_high": {"type": "choice", "choice": "a", "answer_confidence": 0.92},
        "q_low": {"type": "choice", "choice": "b", "answer_confidence": 0.45},
    }
}]

# No gate: the payload is untouched. Not a sentinel, not a flag -- byte-for-byte what the caller
# got before this helper existed, so no existing caller has to learn a field to keep working.
off = fresh()
before = copy.deepcopy(off)
apply_confidence_gate(off, None)
check("gate/an ungated call writes nothing at all", off, before)
for qid, a in off[0]["answers"].items():
    check_true("gate/ungated %s reports no state" % qid, "abstention" not in a)
    check_true("gate/ungated %s carries no threshold" % qid, "abstention_threshold" not in a)
    check_true("gate/ungated %s is not a flag" % qid, "low_confidence" not in a)

# A gate that ran: the answer that cleared it says `passed`, which is the state the boolean
# could not express. This is the whole point -- before, this answer was byte-identical to the
# ungated one above.
on = fresh()
apply_confidence_gate(on, 0.80)
check("gate/cleared answer says passed", on[0]["answers"]["q_high"]["abstention"], GATE_PASSED)
check("gate/cleared answer is still unflagged", "low_confidence" in on[0]["answers"]["q_high"], False)
check("gate/cleared answer echoes the threshold", on[0]["answers"]["q_high"]["abstention_threshold"], 0.80)
check("gate/below-threshold answer says abstained", on[0]["answers"]["q_low"]["abstention"], GATE_ABSTAINED)
check("gate/below-threshold answer is flagged", on[0]["answers"]["q_low"]["low_confidence"], True)
check("gate/below-threshold echoes the threshold", on[0]["answers"]["q_low"]["abstention_threshold"], 0.80)
check_true("gate/the raw answer is untouched",
           on[0]["answers"]["q_high"]["choice"] == "a" and on[0]["answers"]["q_low"]["choice"] == "b")

# The two states a caller used to be unable to tell apart are now told apart by a value, and the
# distinction survives a round trip through JSON, which is how a shadow record reaches a log.
import json as _json  # noqa: E402

round_tripped = _json.loads(_json.dumps(on[0]["answers"]))
check("gate/survives a JSON round trip", round_tripped["q_high"]["abstention"], GATE_PASSED)

# One rule, one implementation: the flag stays `flag_low_confidence`'s, not a second copy of it.
# `apply_confidence_gate(results, 0.0)` is a gate that ran and nothing can fail, which is why
# `flag_low_confidence`'s 0.0 no-op above is still correct and still asserted.
zero = fresh()
apply_confidence_gate(zero, 0.0)
check("gate/0.0 reports passed", zero[0]["answers"]["q_low"]["abstention"], GATE_PASSED)
check_true("gate/0.0 still flags nothing", "low_confidence" not in zero[0]["answers"]["q_low"])

# A non-finite or missing confidence is skipped by `flag_low_confidence`, so it is not an
# abstention. It is also not a pass: the gate ran and could not decide, which is the state Argo
# Rollouts ships as `Inconclusive` and the one collapsing it into success would be the same lie
# `low_confidence`'s absence already tells.
odd = [{"answers": {
    "q_nan": {"type": "choice", "answer_confidence": float("nan")},
    "q_none": {"type": "choice"},
    "q_inf": {"type": "choice", "answer_confidence": float("inf")},
    "q_bool": {"type": "choice", "answer_confidence": True},
    # The fallback path, which is where the number used to leak. With no usable
    # `answer_confidence` the gate reads the entropy `confidence`, and an entropy NaN has to come
    # back as "nothing to gate on" -- not as the NaN itself, which is not None and would
    # therefore read as a pass. `flag_low_confidence` never noticed (NaN < x is False, and
    # None is not None is also False), so nothing else pinned this down.
    "q_nan_entropy": {"type": "choice", "confidence": float("nan")},
    "q_bool_entropy": {"type": "choice", "confidence": True},
    # Both readers above end in `math.isfinite`, and a 400-digit integer is what JSON's unbounded
    # grammar hands them: the gate crashed instead of reporting `unevaluated`, from the same field
    # it is defined against and from the entropy fallback that already leaked a NaN.
    "q_overflow": {"type": "choice", "answer_confidence": 10 ** 400},
    "q_overflow_entropy": {"type": "choice", "confidence": 10 ** 400},
    "q_fine": {"type": "choice", "answer_confidence": 0.99},
}}]
apply_confidence_gate(odd, 0.80)
for qid, a in odd[0]["answers"].items():
    if qid == "q_fine":
        check("gate/a usable confidence is evaluated %s" % qid, a["abstention"], GATE_PASSED)
    else:
        check("gate/unusable confidence is unevaluated %s" % qid, a["abstention"], GATE_UNEVALUATED)
    check_true("gate/unusable confidence is not flagged %s" % qid, "low_confidence" not in a)

# `answer_confidence` is the quantity the gate reads, so a `noul`-style entropy `confidence`
# below threshold does not abstain while `answer_confidence` above it does. The state follows
# the same field the flag follows.
pref = [{"answers": {
    "q_a": {"type": "noul", "answer_confidence": 0.91, "confidence": 0.10},
    "q_b": {"type": "choice", "answer_confidence": 0.10, "confidence": 0.99},
}}]
apply_confidence_gate(pref, 0.80)
check("gate/gate reads answer_confidence (a)", pref[0]["answers"]["q_a"]["abstention"], GATE_PASSED)
check("gate/gate reads answer_confidence (b)", pref[0]["answers"]["q_b"]["abstention"], GATE_ABSTAINED)

# The guards the six call sites rely on: a result that is not a dict, answers that are not a
# dict, and answers that are not dicts. Every call site passes a list it has already partly
apply_confidence_gate([None, {"answers": None}, {"answers": [1, 2]}], 0.5)
apply_confidence_gate([], None)
PASS.append("gate/non-dict results are skipped without raising")

# --------------------------------------------------------------- BUG-005 regression: stale low_confidence across reuse (#910)
# `flag_low_confidence` and `apply_confidence_gate` used to only set `low_confidence: True` and
# never clear it. When a result dict was re-evaluated with a more permissive threshold (or 0.0),
# the stale flag remained, causing downstream callers to treat the answer as permanently abstained.

# 1. flag_low_confidence clears stale flag on re-evaluation
reuse_res = [{
    "answers": {
        "intent": {"choice": "refund", "answer_confidence": 0.8},
    }
}]
reuse_ans = reuse_res[0]["answers"]["intent"]
flag_low_confidence(reuse_res, 0.9)
check_true("stale/flag strict threshold sets flag", reuse_ans.get("low_confidence") is True)
flag_low_confidence(reuse_res, 0.5)
check_true("stale/flag permissive threshold clears flag", "low_confidence" not in reuse_ans)

clean_res = [{"answers": {"q": {"answer_confidence": 0.95}}}]
flag_low_confidence(clean_res, 0.5)
check_true("stale/unflagged answer stays clean", "low_confidence" not in clean_res[0]["answers"]["q"])

# `0.0` belongs to this case as much as `0.5` does -- it is the most permissive threshold there is --
# but the function used to short-circuit on it before the clearing pass, so a stricter threshold's
# marker survived a re-evaluation at `0.0`.
zero_res = [{"answers": {"intent": {"choice": "refund", "answer_confidence": 0.8}}}]
zero_ans = zero_res[0]["answers"]["intent"]
flag_low_confidence(zero_res, 0.9)
check_true("stale/flag 0.9 sets flag before the 0.0 pass", zero_ans.get("low_confidence") is True)
flag_low_confidence(zero_res, 0.0)
check_true("stale/flag 0.0 clears flag", "low_confidence" not in zero_ans)

# One rule, one implementation: the boolean the flag writes and the state the gate reports cannot
# disagree about the same threshold change, which is the claim `apply_confidence_gate`'s docstring
# makes about delegating rather than re-implementing.
gate_both = [{"answers": {"intent": {"choice": "refund", "answer_confidence": 0.8}}}]
apply_confidence_gate(gate_both, 0.9)
apply_confidence_gate(gate_both, 0.0)
check_true("stale/flag and gate agree after 0.9 -> 0.0",
           ("low_confidence" in zero_ans) == ("low_confidence" in gate_both[0]["answers"]["intent"]),
           repr((zero_ans, gate_both[0]["answers"]["intent"])))

# 2. apply_confidence_gate clears stale flag and updates abstention state on re-evaluation
gate_reuse = [{
    "answers": {
        "intent": {"choice": "refund", "answer_confidence": 0.8},
    }
}]
g_ans = gate_reuse[0]["answers"]["intent"]
apply_confidence_gate(gate_reuse, 0.9)
check("stale/gate strict threshold abstains", g_ans["abstention"], GATE_ABSTAINED)
check_true("stale/gate strict threshold sets low_conf", g_ans.get("low_confidence") is True)

apply_confidence_gate(gate_reuse, 0.5)
check("stale/gate permissive threshold passes", g_ans["abstention"], GATE_PASSED)
check_true("stale/gate permissive threshold clears low_conf", "low_confidence" not in g_ans)
check("stale/gate echoes updated threshold", g_ans["abstention_threshold"], 0.5)

apply_confidence_gate(gate_reuse, 0.0)
check("stale/gate 0.0 passes", g_ans["abstention"], GATE_PASSED)
check_true("stale/gate 0.0 clears low_conf", "low_confidence" not in g_ans)

# 3. unusable confidence clears stale low_confidence and marks unevaluated
odd_reuse = [{"answers": {"q": {"choice": "x", "answer_confidence": 0.2, "low_confidence": True}}}]
odd_reuse[0]["answers"]["q"]["answer_confidence"] = float("nan")
apply_confidence_gate(odd_reuse, 0.5)
check("stale/unusable conf marks unevaluated", odd_reuse[0]["answers"]["q"]["abstention"], GATE_UNEVALUATED)
check_true("stale/unusable conf clears stale low_conf", "low_confidence" not in odd_reuse[0]["answers"]["q"])

# --------------------------------------------------------------- the operator's three questions
# End to end through a real call site, weight-free. `decide` forwards to any runner with a
# `predict`, so this exercises `laya/structured.py`'s own gate call site rather than the helper
# in isolation -- the six call sites are where the state used to be skipped entirely.
from laya.structured import decide  # noqa: E402

_GATE_Q = {"type": "choice", "instructions": "Pick one.",
           "criteria": {"a": "alpha", "b": "beta"}}
CONFIDENCES = {"high": 0.95, "low": 0.40}
QUESTIONS = {qid: dict(_GATE_Q) for qid in CONFIDENCES}


class _StubRunner:
    def __init__(self, conf):
        self.conf = conf

    def predict(self, state, questions, **kwargs):
        return {"model": "stub", "answers": {
            qid: {"type": "choice", "choice": "a", "answer_confidence": self.conf[qid]}
            for qid in questions}}


stub = _StubRunner(CONFIDENCES)

# 1. Was the gate in effect at all? Before, both answers were byte-identical to the gated pair
#    below, so the answer was "no way to tell". Now the ungated run simply has no `abstention`
#    key, and the gated one does -- which is how a caller tells them apart.
ungated = decide(stub, "state", questions=QUESTIONS)
for qid in CONFIDENCES:
    check_true("workflow/ungated %s reports no state" % qid, "abstention" not in ungated[qid])

gated = decide(stub, "state", questions=QUESTIONS, min_confidence=0.80)
check("workflow/the cleared answer reports passed", gated["high"]["abstention"], GATE_PASSED)
check("workflow/the low answer reports abstained", gated["low"]["abstention"], GATE_ABSTAINED)

# 2. What fraction abstained, over whatever decisions the operator kept? Computable from the
#    artifact, with no re-run and no out-of-band record of which threshold was used.
log = [gated, decide(stub, "other", questions=QUESTIONS, min_confidence=0.80)]
answers = [a for d in log for a in d.values()]
abstained = sum(a["abstention"] == GATE_ABSTAINED for a in answers)
check("workflow/abstention rate over the log", abstained, 2)
check("workflow/…out of", len(answers), 4)
check_true("workflow/…and no answer is left unlabelled", all("abstention" in a for a in answers))

# 3. What threshold produced these? `flag_low_confidence` consumes it and drops it, so before
#    this a batch that gated different request classes differently could not be re-split.
check("workflow/the threshold is recoverable", gated["low"]["abstention_threshold"], 0.80)
check_true("workflow/…and a per-class mix is re-splittable",
           {a["abstention_threshold"] for a in gated.values()} == {0.80})

# The schema projection is unchanged -- an abstained field is still `null` -- so the flat
# `values` mapping still cannot say why. `return_details=True` is where the state is readable,
# and it was unreadable before.
SCHEMA = {"type": "object",
          "properties": {"high": {"type": "string", "enum": ["a", "b"]},
                         "low": {"type": "string", "enum": ["a", "b"]}}}
projected = decide(stub, "state", schema=SCHEMA, min_confidence=0.80)
check("workflow/the schema projection is unchanged", projected["low"], None)
check("workflow/…and still reports the cleared field", projected["high"], "a")
detailed = decide(stub, "state", schema=SCHEMA, min_confidence=0.80, return_details=True)
check("workflow/return_details carries the state",
      detailed.answers["low"]["abstention"], GATE_ABSTAINED)

# --------------------------------------------------------------- exported
import laya  # noqa: E402

check_true("export/answer_confidence is importable from laya", hasattr(laya, "answer_confidence"))
check_true("export/answer_confidence is in __all__", "answer_confidence" in laya.__all__)
check_true("export/answer_confidence is in dir()", "answer_confidence" in dir(laya))
check_true("export/check_min_confidence is importable from laya", hasattr(laya, "check_min_confidence"))
check_true("export/check_min_confidence is in __all__", "check_min_confidence" in laya.__all__)
check_true("export/check_min_confidence is in dir()", "check_min_confidence" in dir(laya))
check_true("export/flag_low_confidence is importable from laya", hasattr(laya, "flag_low_confidence"))
check_true("export/flag_low_confidence is in __all__", "flag_low_confidence" in laya.__all__)
check_true("export/flag_low_confidence is in dir()", "flag_low_confidence" in dir(laya))
check_true("export/apply_confidence_gate is importable from laya", hasattr(laya, "apply_confidence_gate"))
check_true("export/apply_confidence_gate is in __all__", "apply_confidence_gate" in laya.__all__)
check_true("export/apply_confidence_gate is in dir()", "apply_confidence_gate" in dir(laya))
check_true("export/GATE_STATES is importable from laya", hasattr(laya, "GATE_STATES"))
check_true("export/GATE_STATES is in __all__", "GATE_STATES" in laya.__all__)
check_true("export/GATE_STATES is in dir()", "GATE_STATES" in dir(laya))
# The three values are reached through GATE_STATES rather than exported one by one: iterating the
# vocabulary is the whole use, and every `laya.__all__` name has to earn a documented entry
# (`tests/test_packaging.py` enforces that).
import laya.confidence as _confidence  # noqa: E402

check("export/GATE_STATES is the vocabulary a caller iterates", list(GATE_STATES),
      [_confidence.GATE_PASSED, _confidence.GATE_ABSTAINED, _confidence.GATE_UNEVALUATED])
check("export/laya re-exports the same tuple", laya.GATE_STATES, _confidence.GATE_STATES)

# --------------------------------- the two doc pages must attribute confidence per question type
# `docs/structured.md` and `docs/questions-and-answers.md` each opened their confidence section with
# "`confidence` is normalized entropy". Neither agent writes that for every type:
# `Agent._decode_answers` (`laya/agent.py::Agent._decode_answers`) and `ONNXAgent._decode_answers`
# (`laya/onnx_agent.py::ONNXAgent._decode_answers`) put `confidence_from_probs(p, k)` =
# `1 - H(p) / log(k)` in the
# `choice` and `score` answers and `max(p_true, 1 - p_true)` in the `noul` one. Each page printed a
# `noul` in the very block its sentence introduces, so the sentence was falsifiable from the page
# alone: the Q&A page's sample answer is `{"type": "noul", "noul": 0.8727, "confidence": 0.8727}`,
# where entropy over two options reads 0.45, and structured.md's `Ticket` carries `needs_human: bool`
# beside the `department` choice. structured.md also quoted `0.71` for the field whose probabilities it
# prints one line below as `{"billing": 0.94, "support": 0.06, "sales": 0.0}` -- 0.79 under the formula
# the page names. So both pages are read as data here: every number they print is recomputed from the
# distribution printed beside it, the option count from the schema's own labels, and each formula has
# to appear in a sentence naming exactly the types the code uses it for. No weights, no pydantic.
import ast as _ast  # noqa: E402  (`ast` itself is imported again further down)
import re as _re  # noqa: E402  (same reason: this section must not rebind `re`)

from laya.structured import questions_from_json_schema  # noqa: E402

DOC_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DOC_STRUCTURED = os.path.join(DOC_ROOT, "docs", "structured.md")
DOC_QA = os.path.join(DOC_ROOT, "docs", "questions-and-answers.md")
DOC_AGENT_PY = os.path.join(DOC_ROOT, "laya", "agent.py")
DOC_ONNX_PY = os.path.join(DOC_ROOT, "laya", "onnx_agent.py")

ATTRIBUTED = {"entropy": ["choice", "score"], "maxp": ["noul"]}

# A type has to be named in the same sentence as the formula, so `choice`/`score` cannot inherit the
# `noul` formula and back. Only backticked type tokens count: the pages use "the choice is explicit" as
# English, and a gate that read that as a question type would pass the wrong page.
ENTROPY_MENTION = _re.compile(r"normalized\s+entropy")
MAXP_MENTION = _re.compile(r"max\(\s*p(?:_true)?\s*,\s*1\s*-\s*p(?:_true)?\s*\)")
TYPE_TOKEN = _re.compile(r"`(choice|score|noul)`")

# The blanket claims, and the one number that was simply wrong.
BLANKET_ENTROPY = _re.compile(r"`?confidence`?\s+is\s+normalized\s+entropy", _re.I)
BLANKET_SAMPLE_NOTE = _re.compile(r"normalized\s+entropy,\s+which\s+depends\s+on\s+label\s+count")
MAXP_IN_CODE = _re.compile(r"max\(\s*float\(\s*p\[1\]\s*\)\s*,\s*1\.0\s*-\s*float\(\s*p\[1\]\s*\)\s*\)")

# The pages as they ship on main, sentence for sentence. Every ban below fires on this text and every
# attribution rule below shows this text failing it, so no rule here is a guess about what changed.
OLD_BLANKETS = (
    ("docs/structured.md",
     "`confidence`. `confidence` is normalized entropy, which depends on how many options the "
     "question had: `tests/test_confidence.py` pins that a two-option distribution comes back as "
     "0.90 on a `noul` and 0.53 on an equivalent `choice`, so it does not compare against a "
     "threshold."),
    ("docs/questions-and-answers.md",
     "`confidence` is normalized entropy: high when the distribution is peaked, low when it is "
     "spread out, regardless of whether the top answer is correct."),
)
OLD_SAMPLE_LINE = ('result.confidence["department"]        '
                   '# 0.71  normalized entropy, which depends on label count')


def _read_page(path):
    # newline="" plus an explicit encoding: the pages carry em-dashes, and a CRLF checkout or a
    # non-UTF-8 locale must not change what the rules below see.
    with open(path, encoding="utf-8", newline="") as fh:
        return fh.read().replace("\r\n", "\n")


def _flatten(text):
    return " ".join(text.split())


def _fenced_blocks(text):
    """Every fenced block on a page, in order, as (language, body)."""
    blocks, lang, body = [], None, []
    for line in text.split("\n"):
        stripped = line.strip()
        if lang is None and stripped.startswith("```"):
            lang = stripped[3:].strip()
            body = []
        elif lang is not None and stripped.startswith("```"):
            blocks.append((lang, "\n".join(body)))
            lang = None
        elif lang is not None:
            body.append(line)
    assert lang is None, "unbalanced fence in %s" % text[:40]
    return blocks


def _prose(text):
    """The page's prose: fenced blocks dropped, line wraps undone, whitespace collapsed."""
    out, in_fence = [], False
    for line in text.split("\n"):
        if line.strip().startswith("```"):
            in_fence = not in_fence
            continue
        if not in_fence:
            out.append(line)
    return _flatten("\n".join(out))


def _attributions(prose):
    """{formula: sorted question types named in the same sentence as it}."""
    got = {}
    for sentence in _re.split(r"(?<=[.!?])\s+", prose):
        types = set(TYPE_TOKEN.findall(sentence))
        if not types:
            continue
        if ENTROPY_MENTION.search(sentence):
            got.setdefault("entropy", set()).update(types)
        if MAXP_MENTION.search(sentence):
            got.setdefault("maxp", set()).update(types)
    return {key: sorted(value) for key, value in sorted(got.items())}


def _decode_conf_formulas(path):
    """{question type: the formula its `confidence` is built from}, read out of that agent's own
    `_decode_answers` with `ast`: `entropy` where the answer dict calls `confidence_from_probs`,
    `maxp` where it takes `max(float(p[1]), 1.0 - float(p[1]))`. A bare name resolves one hop to its
    assignment in the same function, because the ONNX builder binds `conf_score` once and uses it for
    both of its entropy types."""
    src = _read_page(path)
    fn = next((node for node in _ast.walk(_ast.parse(src))
               if isinstance(node, _ast.FunctionDef) and node.name == "_decode_answers"), None)
    assert fn is not None, "%s no longer defines _decode_answers" % path
    bound = {}
    for node in _ast.walk(fn):
        if (isinstance(node, _ast.Assign) and len(node.targets) == 1
                and isinstance(node.targets[0], _ast.Name)):
            bound[node.targets[0].id] = _ast.get_source_segment(src, node.value) or ""
    got = {}
    for node in _ast.walk(fn):
        if not isinstance(node, _ast.Dict):
            continue
        pairs = {}
        for key, value in zip(node.keys, node.values):
            if isinstance(key, _ast.Constant) and isinstance(key.value, str):
                pairs[key.value] = value
        if "confidence" not in pairs or "type" not in pairs:
            continue
        qtype = pairs["type"].value
        expr = pairs["confidence"]
        text = bound.get(expr.id, "") if isinstance(expr, _ast.Name) else (
            _ast.get_source_segment(src, expr) or "")
        if "confidence_from_probs" in text:
            formula = "entropy"
        elif MAXP_IN_CODE.search(text):
            formula = "maxp"
        else:
            raise AssertionError("%s builds `%s`'s confidence from %r" % (path, qtype, text))
        assert qtype not in got, "%s decodes `%s` twice" % (path, qtype)
        got[qtype] = formula
    return got


def _structured_sample():
    """The `return_details=True` block as data: the number printed for each field, the probabilities
    printed for the same field, and the option count named beside the confidence number."""
    src = _read_page(DOC_STRUCTURED)

    def printed(key):
        m = _re.search(r'result\.' + key + r'\["department"\]\s*#\s*([0-9.]+)', src)
        assert m, 'the sample block no longer prints result.%s["department"]' % key
        return float(m.group(1))

    m = _re.search(r'result\.probabilities\["department"\]\s*#\s*(\{[^}]*\})', src)
    assert m, "the sample block no longer prints the probabilities the two numbers come from"
    probs = _ast.literal_eval(m.group(1))
    note = _re.search(r'result\.confidence\["department"\]\s*#\s*[0-9.]+\s+([^\n]*)', src)
    assert note, "the confidence line lost its note"
    k = _re.search(r"(\d+)-option", note.group(1))
    assert k, "the confidence line must name the option count its formula depends on: %r" % note.group(1)
    return printed("confidence"), printed("answer_confidence"), probs, int(k.group(1)), note.group(1)


def test_no_page_calls_confidence_one_formula():
    for path in (DOC_STRUCTURED, DOC_QA):
        prose = _prose(_read_page(path))
        assert not BLANKET_ENTROPY.search(prose), "%s still says `confidence` is entropy" % path
    src = _read_page(DOC_STRUCTURED)
    assert not BLANKET_SAMPLE_NOTE.search(src), "the sample note still blames the label count alone"
    for name, old in OLD_BLANKETS:
        assert BLANKET_ENTROPY.search(_flatten(old)), "the ban does not fire on %s's old sentence" % name
    assert BLANKET_SAMPLE_NOTE.search(OLD_SAMPLE_LINE), "the sample-note ban does not fire on main's line"


def test_each_page_attributes_both_formulas_to_named_types():
    for path in (DOC_STRUCTURED, DOC_QA):
        got = _attributions(_prose(_read_page(path)))
        assert got == ATTRIBUTED, "%s attributes %s, want %s" % (path, got, ATTRIBUTED)
    # main's prose satisfies neither half, so the rule could not have passed before the fix.
    assert _attributions(_flatten(OLD_BLANKETS[1][1])) == {}, "main's Q&A sentence passes the rule"
    assert _attributions(_flatten(OLD_BLANKETS[0][1])) == {"entropy": ["choice", "noul"]}, (
        "main's structured.md sentence passes the rule")


def test_the_agents_build_what_the_pages_attribute():
    want = {}
    for path in (DOC_AGENT_PY, DOC_ONNX_PY):
        got = _decode_conf_formulas(path)
        assert sorted(got) == ["choice", "noul", "score"], "%s decodes %s" % (path, sorted(got))
        assert got == {"choice": "entropy", "score": "entropy", "noul": "maxp"}, got
        want = got
    derived = {"entropy": sorted(t for t, f in want.items() if f == "entropy"),
               "maxp": sorted(t for t, f in want.items() if f == "maxp")}
    assert derived == ATTRIBUTED, "the pages are held to a transcription, not to the agents: %s" % derived
    for path in (DOC_STRUCTURED, DOC_QA):
        got = _attributions(_prose(_read_page(path)))
        assert got == derived, "%s attributes %s; the agents build %s" % (path, got, derived)


def test_structured_page_sample_numbers_recompute():
    conf, answer_conf, probs, k, note = _structured_sample()
    p = np.array([probs[label] for label in probs])
    assert k == len(p), "the page says %d-option and prints %d labels" % (k, len(p))
    assert conf == round(confidence_from_probs(p, k), 2), (
        "the page prints confidence %s for %s over %d options; that formula gives %s"
        % (conf, probs, k, round(confidence_from_probs(p, k), 2)))
    assert abs(answer_conf - float(p.max())) < 5e-3, (
        "answer_confidence %s is not max(p)=%s of the probabilities printed beside it"
        % (answer_conf, round(float(p.max()), 4)))
    assert "`choice`" in note, "the confidence line must say which type the field is: %r" % note
    # and main's number was not this formula's either way: the witness stays true while the
    # probabilities on the page stay these.
    assert abs(0.71 - round(confidence_from_probs(p, k), 2)) > 5e-3, (
        "0.71 now recomputes, so this witness has gone stale: %s" % probs)


def test_structured_page_claims_a_schema_that_mixes_the_two_types():
    """The page says its own `Ticket` holds one `choice` field and one `noul` field, which is what
    makes one `confidence` dict carry two scales. The compiler that builds those questions has to
    agree, and the probabilities in the sample have to be that field's labels."""
    src = _read_page(DOC_STRUCTURED)
    ticket = next((body for _lang, body in _fenced_blocks(src) if "class Ticket" in body), None)
    assert ticket is not None, "the page lost the Ticket schema the confidence sample reads"
    fields = dict(_re.findall(r"^\s*(\w+)\s*:\s*([^\n]+?)\s*$", ticket, _re.M))
    assert sorted(fields) == ["department", "needs_human", "urgency"], fields
    labels = [s.strip().strip("'\"")
              for s in _re.search(r"Literal\[(.*?)\]", fields["department"]).group(1).split(",")]
    assert fields["needs_human"] == "bool", fields
    _conf, _ac, probs, _k, _note = _structured_sample()
    assert labels == list(probs), "the sample's probabilities are not the schema's labels"
    compiled = questions_from_json_schema({
        "type": "object",
        "properties": {
            "department": {"enum": labels},
            "needs_human": {"type": "boolean"},
        },
    })
    got = {qid: q["type"] for qid, q in compiled.items()}
    assert got == {"department": "choice", "needs_human": "noul"}, got


def _qa_answers():
    """{question type: the sample answer the page prints for it}. Each block is a dict literal, so the
    numbers are read as data and not matched as prose."""
    out = {}
    for _lang, body in _fenced_blocks(_read_page(DOC_QA)):
        try:
            node = _ast.literal_eval(body.strip())
        except (SyntaxError, ValueError):
            continue
        if isinstance(node, dict) and "confidence" in node and "answer_confidence" in node:
            out[node["type"]] = node
    return out


def test_questions_page_sample_answers_recompute():
    answers = _qa_answers()
    assert sorted(answers) == ["choice", "noul", "score"], (
        "the page must show one sample answer per question type, it shows %s" % sorted(answers))
    for qtype in ("choice", "score"):
        ans = answers[qtype]
        p = np.array(list(ans["probabilities"].values()))
        entropy = confidence_from_probs(p, len(p))
        assert abs(ans["confidence"] - entropy) <= 1e-3, (
            "%s: printed confidence %s, entropy of %s is %s"
            % (qtype, ans["confidence"], list(ans["probabilities"]), round(entropy, 4)))
        assert ans["answer_confidence"] == round(float(p.max()), 4), (
            qtype, ans["answer_confidence"], round(float(p.max()), 4))
    ans = answers["noul"]
    p_true = ans["noul"]
    assert ans["confidence"] == round(max(p_true, 1.0 - p_true), 4), ans
    assert ans["confidence"] == ans["answer_confidence"], (
        "over two options the two are the same number, so a mismatch means the sample changed shape")
    entropy2 = round(confidence_from_probs(np.array([1.0 - p_true, p_true]), 2), 4)
    assert ans["confidence"] != entropy2, (
        "the noul sample now reads as entropy, so the blanket sentence would be defensible")
    m = _re.search(r"rather than the ([0-9.]+)", _prose(_read_page(DOC_QA)))
    assert m, "the page must keep the counterfactual that proves its noul row is not entropy"
    assert abs(float(m.group(1)) - entropy2) < 0.005, (m.group(1), entropy2)


def test_questions_page_row_table_matches_its_samples():
    body = next((b for _l, b in _fenced_blocks(_read_page(DOC_QA)) if "confidence 0." in b), None)
    assert body is not None, "the page lost the three-row confidence/answer_confidence comparison"
    rows = _re.findall(r"^(\w+)\s+confidence\s+([0-9.]+)\s+answer_confidence\s+([0-9.]+)", body, _re.M)
    answers = _qa_answers()
    named = {"dept": "choice", "urgent": "noul", "severity": "score"}
    assert [row[0] for row in rows] == list(named), rows
    for field, conf, ac in rows:
        ans = answers[named[field]]
        assert abs(float(conf) - ans["confidence"]) <= 1e-3, (field, conf, ans["confidence"])
        assert abs(float(ac) - ans["answer_confidence"]) <= 1e-3, (field, ac, ans["answer_confidence"])
    assert sorted(named.values()) == sorted(_decode_conf_formulas(DOC_AGENT_PY)), (
        "the rows do not cover the three types the agents build `confidence` for")


for _fn in (test_no_page_calls_confidence_one_formula,
            test_each_page_attributes_both_formulas_to_named_types,
            test_the_agents_build_what_the_pages_attribute,
            test_structured_page_sample_numbers_recompute,
            test_structured_page_claims_a_schema_that_mixes_the_two_types,
            test_questions_page_sample_answers_recompute,
            test_questions_page_row_table_matches_its_samples):
    try:
        _fn()
    except AssertionError as e:
        FAIL.append("doc-pages/%s: %s" % (_fn.__name__, e))
    except Exception as e:                      # a crash is a failure, never a silent pass
        FAIL.append("doc-pages/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
    else:
        PASS.append("doc-pages/%s" % _fn.__name__)

# ------------------------------------------------- a preset page's conclusions follow their numbers
# `examples/28_presets_moderation.py` prints a summary over `laya.moderation_questions()` answers,
# and the version this section replaces hardcoded three of its conclusions:
#   * `benign -> every flag 0.000, severity 0.24 / 3` printed `toxic` alone, while the same page's
#     table showed the benign post reading 0.066 on `spam`;
#   * ``severity orders the set correctly (1.34 > 1.05 > 0.99 > 0.24)`` put the `>` signs in the
#     format string, so the sentence stayed "correct" whatever the scores came back as;
#   * `the questions that separate them are harassment and threat` named two of four flags, while
#     the gaps measured on that same run were harassment 0.518, spam 0.193, threat 0.054 -- threat
#     was third, and spam was never mentioned.
# The page now builds those sentences from pure functions over its answers. No weights are loaded:
# the functions are extracted from the example's own AST and exec'd here on fabricated answers, so
# the gate drives the code the page actually runs rather than re-reading its prose.
import ast  # noqa: E402
import builtins  # noqa: E402
import re  # noqa: E402

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
EXAMPLE_28 = os.path.join(ROOT, "examples", "28_presets_moderation.py")
HELPERS_28 = ("noul_flags", "score_ceiling", "flag_line", "past_half", "gaps", "rank_by_severity")

# The page as it ships on main: the banner's universality claim and the three conclusions, as
# literal source lines. Every ban below is witnessed against this text, and every positive rule
# below is witnessed by showing this text does not satisfy it.
OLD_PAGE = '''
    lights up harassment, a generic one only toxic, spam only spam, and a normal post
    nothing at all.
        label, a["toxic"]["noul"], a["harassment"]["noul"], a["threat"]["noul"],
print("   benign          -> every flag %.3f, severity %.2f / 3"
      % (ben["toxic"]["noul"], ben["severity"]["score"]))
print("   `severity` orders the set correctly (%.2f > %.2f > %.2f > %.2f) but is a coarse 0-3"
print("   posts differ by only %.3f on `toxic`: the questions that separate them are")
print("   `harassment` and `threat`, not `toxic` on its own.")
'''

UNIVERSAL_ONE_FLOAT = re.compile(r"every (?:flag|score|level)[^.]{0,12}%\.\d+f", re.I)
ASSERTED_CHAIN = re.compile(r"%\.2f\s*>\s*%\.2f")
NAMED_SEPARATORS = re.compile(r"separat\w+[^.]{0,60}`(?:toxic|harassment|threat|spam)`"
                              r"[^.]{0,30}`(?:toxic|harassment|threat|spam)`", re.I)
HARDCODED_FLAG_READ = re.compile(r"\[[\"'](?:toxic|harassment|threat|spam)[\"']\]\[[\"']noul[\"']\]")
NOTHING_QUALIFIED = re.compile(r"nothing past 0\.5[^,]{0,10}, not zero", re.I)
DERIVED_FLAGS = re.compile(r"q\[.type.\]\s*==\s*[\"']noul[\"']")
DERIVED_CEILING = re.compile(r"len\(q\[.criteria.\]\)\s*-\s*1")


def _src28():
    with open(EXAMPLE_28, encoding="utf-8") as fh:
        return fh.read()


def _helpers28():
    """Exec only the example's top-level helper functions -- its `load()` call needs weights."""
    tree = ast.parse(_src28())
    nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in HELPERS_28]
    ns = {}
    exec(compile(ast.Module(body=nodes, type_ignores=[]), EXAMPLE_28, "exec"), ns)  # noqa: S102
    return ns


def _called28():
    """Names the example's top-level *statements* use, so a helper cannot pass by being dead."""
    tree = ast.parse(_src28())
    top = [n for n in tree.body if not isinstance(
        n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Import, ast.ImportFrom))]
    return {node.id for stmt in top for node in ast.walk(stmt)
            if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)}


def _a(toxic, harassment, threat, spam, severity):
    return {"toxic": {"noul": toxic}, "harassment": {"noul": harassment},
            "threat": {"noul": threat}, "spam": {"noul": spam},
            "severity": {"score": severity}}


def test_preset_is_four_flags_and_one_rubric():
    """The docstring and the example's banner both claim this shape; nothing asserted it."""
    mq = laya.moderation_questions()
    assert list(mq) == ["toxic", "harassment", "threat", "spam", "severity"], list(mq)
    assert {k: q["type"] for k, q in mq.items()} == {
        "toxic": "noul", "harassment": "noul", "threat": "noul", "spam": "noul",
        "severity": "score"}
    assert len(mq["severity"]["criteria"]) == 4, len(mq["severity"]["criteria"])


def _free_names(node):
    """Names a function reads that it neither binds locally nor takes as an argument."""
    bound = {a.arg for fn in [node] + [k for k in ast.walk(node) if isinstance(k, ast.Lambda)]
             for a in fn.args.args}
    bound |= {n.id for n in ast.walk(node)
              if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Store)}
    return {n.id for n in ast.walk(node)
            if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Load) and n.id not in bound}


def test_helpers_are_live_and_pure():
    """Each helper is called by the page, takes its whole world as arguments, and returns."""
    ns = _helpers28()
    got = sorted(k for k in ns if not k.startswith("__"))
    assert got == sorted(HELPERS_28), "example 28 lost a helper: %s" % got
    unused = set(HELPERS_28) - _called28()
    assert not unused, "defined but never called at module level: %s" % sorted(unused)
    for name in HELPERS_28:
        node = _fn_node(name)
        outside = sorted(n for n in _free_names(node) if not hasattr(builtins, n))
        assert not outside, "%s reads %s from the page, so this gate cannot drive it" % (name, outside)
        prints = [c for c in ast.walk(node)
                  if isinstance(c, ast.Call) and isinstance(c.func, ast.Name) and c.func.id == "print"]
        assert not prints, "%s prints instead of returning, so the summary is not checkable" % name


def _fn_node(name):
    for n in ast.parse(_src28()).body:
        if isinstance(n, ast.FunctionDef) and n.name == name:
            return n
    raise KeyError(name)


def test_flag_line_prints_every_flag():
    ns = _helpers28()
    flags = ns["noul_flags"](laya.moderation_questions())
    assert flags == ["toxic", "harassment", "threat", "spam"], flags
    line = ns["flag_line"](_a(0.0, 0.0, 0.0, 0.066, 0.24), flags)
    assert "0.066" in line, "the non-zero flag vanished: %r" % line
    assert re.findall(r"`([a-z]+)`", line) == flags, line
    assert len(re.findall(r"\d\.\d{3}", line)) == 4, "one number per flag: %r" % line


def test_past_half_uses_describes_cut():
    """`_common.describe` labels a noul true at `> 0.5`; the page must agree with it exactly."""
    ns = _helpers28()
    flags = ns["noul_flags"](laya.moderation_questions())
    assert ns["past_half"](_a(0.0, 0.0, 0.0, 0.066, 0.24), flags) == []
    assert ns["past_half"](_a(0.587, 0.662, 0.142, 0.049, 1.34), flags) == ["toxic", "harassment"]
    assert ns["past_half"](_a(0.5, 0.5, 0.5, 0.5, 1.0), flags) == [], "0.5 reads false, not true"


def test_gaps_names_the_widest_separators():
    """The old page said harassment and threat; on these numbers it is harassment and spam."""
    ns = _helpers28()
    flags = ns["noul_flags"](laya.moderation_questions())
    tgt, gen = _a(0.587, 0.662, 0.142, 0.049, 1.34), _a(0.584, 0.144, 0.088, 0.242, 1.05)
    g = ns["gaps"](tgt, gen, flags)
    assert [k for k, _ in g] == ["harassment", "spam", "threat", "toxic"], g
    assert abs(dict(g)["harassment"] - 0.518) < 1e-9 and dict(g)["spam"] < 0, g
    # and the name follows the data: move `threat` to the top and it is the separator.
    assert ns["gaps"](_a(0.1, 0.1, 0.9, 0.1, 1), _a(0.1, 0.1, 0.1, 0.1, 1), flags)[0][0] == "threat"


def test_rank_cannot_assert_its_own_order():
    ns = _helpers28()
    labels = ["a", "b", "c", "d"]
    scores = {"a": 1.34, "b": 1.05, "c": 0.99, "d": 0.24}
    results = {k: _a(0, 0, 0, 0, v) for k, v in scores.items()}
    chain, in_order = ns["rank_by_severity"](results, labels)
    assert in_order is True, chain
    assert [float(x) for x in re.findall(r"\d+\.\d{2}", chain)] == [1.34, 1.05, 0.99, 0.24], chain

    shuffled = {k: _a(0, 0, 0, 0, scores[j]) for k, j in
                zip(labels, ["d", "a", "c", "b"])}
    chain2, in_order2 = ns["rank_by_severity"](shuffled, labels)
    assert in_order2 is False, "the verdict must flip when the ranking does: %s" % chain2
    assert [float(x) for x in re.findall(r"\d+\.\d{2}", chain2)] == [1.34, 1.05, 0.99, 0.24], (
        "the printed chain must descend, whatever the order it lists: %s" % chain2)
    assert chain2.index("b") < chain2.index("a"), chain2


def test_page_drops_the_hardcoded_conclusions():
    """Each ban fires on main's page and not on this one -- a ban with no witness is a guess."""
    src = _src28()
    for name, rule in (("every flag %.3f", UNIVERSAL_ONE_FLOAT),
                       ("%.2f > %.2f in a format string", ASSERTED_CHAIN),
                       ("separate them are `harassment` and `threat`", NAMED_SEPARATORS),
                       ('a["toxic"]["noul"] in the summary', HARDCODED_FLAG_READ)):
        assert rule.search(OLD_PAGE), "%s does not fire on the wording it bans" % name
        assert not rule.search(src), "%s is still in example 28" % name


def test_page_qualifies_the_banner_and_derives_its_numbers():
    """The positive half: the page must read its flag list and rubric size from the preset."""
    src = _src28()
    assert NOTHING_QUALIFIED.search(src), "the banner must say that `nothing` is not zero"
    assert not re.search(r"nothing at all", src, re.I), "the unqualified claim is back"
    assert DERIVED_FLAGS.search(src), "flags must be read off the preset, not typed"
    assert DERIVED_CEILING.search(src), "the rubric ceiling must be read off the criteria"
    assert not re.search(r"/ 3\b", src), "a hardcoded severity divisor is back"
    # and main's page satisfies none of that, so these rules could not have passed before.
    assert not NOTHING_QUALIFIED.search(OLD_PAGE)
    assert re.search(r"nothing at all", OLD_PAGE, re.I)
    assert not DERIVED_FLAGS.search(OLD_PAGE) and not DERIVED_CEILING.search(OLD_PAGE)


# ------------------------------- example 03 must give `confidence` separately per question type
# `examples/03_reading_the_result.py` is the page a reader comes back to "when writing your own
# glue code", and its field tour closed with "`confidence` is 1 minus normalised entropy -- a scale
# that moves with the option count on the same answer". `Agent._decode_answers` runs two formulas
# and picks by question type (`laya/agent.py:1366-1402`):
#   * `choice` and `score`  ->  `round(confidence_from_probs(p, k), 4)` = `1 - H(p) / log(k)`
#   * `noul`                ->  `round(max(float(p[1]), 1.0 - float(p[1])), 4)`
# The page calls all three types in one `predict()`, so no single sentence covers `confidence` --
# and the proof is in the run the page itself prints: a `noul` answer reports the same number in
# `confidence` and `answer_confidence`, which a normalized entropy of a two-option distribution
# never equals (0.9/0.1 reads 0.531 as entropy, 0.900 as max(p)). Same defect class as examples
# 40/18/30 and as `DecisionResult`'s docstring, on the page a beginner reads first. The two pages
# that still carry the blanket sentence (`docs/structured.md`, `docs/questions-and-answers.md`) are
# left for a follow-up.
EXAMPLE_03 = os.path.join(ROOT, "examples", "03_reading_the_result.py")
AGENT_PY = os.path.join(ROOT, "laya", "agent.py")
ONNX_PY = os.path.join(ROOT, "laya", "onnx_agent.py")

# The wording this section bans, as it ships on main. Every ban below is witnessed against it, and
# every positive rule below is witnessed by showing that it does not satisfy the rule.
OLD_PAGE_03 = """
   `confidence` is 1 minus normalised entropy -- a scale that moves with the option
   count on the same answer. `answer_confidence` is max(p), the probability mass on
   the answer being reported, and it is the field to gate on (example 18 routes at a
   threshold on it).
"""

BAN_UNIFORM_ENTROPY = re.compile(r"`confidence` is 1 minus normalised entropy -- a scale", re.I)
NAMES_ENTROPY_TYPES = re.compile(r"normalised entropy on a `choice` or `score` answer", re.I)
NAMES_NOUL_FORMULA = re.compile(r"max\(p\[1\],\s*1\s*-\s*p\[1\]\)")
ATTRIBUTES_NOUL = re.compile(r"-- on a `noul`\.")
CALLS_SCALES_INCOMPARABLE = re.compile(r"scales are not comparable", re.I)
NAMES_ANSWER_MAXP = re.compile(r"`answer_confidence` is max\(p\),")
GATES_ON_ANSWER = re.compile(r"field to gate on \(example 18", re.I)
NAMES_BINNING_SCOPE = re.compile(
    r"`binning_map` remaps `answer_confidence` and leaves `confidence` alone", re.I)


def _src03(path):
    with open(path, encoding="utf-8") as fh:
        return fh.read()


def _printed_text(path):
    """The page's output as one flattened string.

    Every string literal handed to `print()` anywhere in the file, joined and whitespace-collapsed,
    so a sentence the page wraps across four `print()` calls is still one sentence to the rules
    below -- and a claim that survives only because it is split across lines cannot hide from them.
    """
    parts = []
    for node in ast.walk(ast.parse(_src03(path))):
        if (isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
                and node.func.id == "print"):
            parts.extend(a.value for a in ast.walk(node)
                         if isinstance(a, ast.Constant) and isinstance(a.value, str))
    return " ".join(" ".join(parts).split())


def _confidence_exprs(path):
    """{answer type: source of the expression that becomes its `confidence`}.

    Read off `_decode_answers` through AST, one entry per answer dict the builder constructs, so
    the page is held to the code that produces the field rather than to a comment about it. A value
    that is a local name (the ONNX path pre-computes `conf_score`) is resolved to what it is bound
    to inside the same function.
    """
    src = _src03(path)
    func = next(n for n in ast.walk(ast.parse(src))
                if isinstance(n, ast.FunctionDef) and n.name == "_decode_answers")
    bound = {}
    for node in ast.walk(func):
        if (isinstance(node, ast.Assign) and len(node.targets) == 1
                and isinstance(node.targets[0], ast.Name)):
            bound[node.targets[0].id] = ast.get_source_segment(src, node.value)
    found = {}
    for node in ast.walk(func):
        if not isinstance(node, ast.Dict):
            continue
        literal = {k.value: v for k, v in zip(node.keys, node.values)
                   if isinstance(k, ast.Constant) and isinstance(k.value, str)}
        if "type" not in literal or "confidence" not in literal:
            continue
        if not isinstance(literal["type"], ast.Constant):
            continue
        expr = ast.get_source_segment(src, literal["confidence"])
        found[literal["type"].value] = bound.get(expr, expr)
    return found


def test_page_drops_the_single_entropy_formula():
    text = _printed_text(EXAMPLE_03)
    assert BAN_UNIFORM_ENTROPY.search(OLD_PAGE_03), "the ban does not fire on the wording it bans"
    assert not BAN_UNIFORM_ENTROPY.search(
        text), "example 03 again gives `confidence` one formula for all three question types"
    for name, rule in (("the entropy formula is attributed to `choice` and `score`",
                        NAMES_ENTROPY_TYPES),
                       ("the `noul` formula is written out", NAMES_NOUL_FORMULA),
                       ("that formula is attributed to `noul`", ATTRIBUTES_NOUL),
                       ("the two scales are called incomparable", CALLS_SCALES_INCOMPARABLE),
                       ("`answer_confidence` is max(p)", NAMES_ANSWER_MAXP),
                       ("the gate field is named with its page", GATES_ON_ANSWER),
                       ("the binning map remaps only `answer_confidence`", NAMES_BINNING_SCOPE)):
        assert rule.search(text), "example 03 no longer states that %s" % name
    # and main's page satisfies none of the new rules, so they could not have passed before.
    old = " ".join(OLD_PAGE_03.split())
    assert not NAMES_ENTROPY_TYPES.search(old) and not NAMES_NOUL_FORMULA.search(old)
    assert not CALLS_SCALES_INCOMPARABLE.search(old) and not NAMES_BINNING_SCOPE.search(old)


def test_page_names_the_formulas_the_agents_build():
    exprs = _confidence_exprs(AGENT_PY)
    assert set(exprs) == {"choice", "score", "noul"}, (
        "the scan reached %s, not all three answer dicts" % sorted(exprs))
    assert all(exprs.values()), "a `confidence` value could not be read: %s" % exprs
    assert "confidence_from_probs" in exprs["choice"], exprs["choice"]
    assert "confidence_from_probs" in exprs["score"], exprs["score"]
    assert "1.0 - float(p[1])" in exprs["noul"], exprs["noul"]
    assert "confidence_from_probs" not in exprs["noul"], (
        "`noul` has switched to entropy -- the page's per-type split is now the stale claim: %s"
        % exprs["noul"])
    text = _printed_text(EXAMPLE_03)
    assert NAMES_ENTROPY_TYPES.search(text) and NAMES_NOUL_FORMULA.search(text), (
        "the page must name both expressions the builder uses")


def test_page_cites_the_numbers_the_repo_functions_produce():
    """The earned-numbers arm: the pairs the page prints must be what `laya.common` returns."""
    text = _printed_text(EXAMPLE_03)
    for p_true in (0.60, 0.90):
        p = np.array([1.0 - p_true, p_true])
        noul = "%.3f" % max(float(p[1]), 1.0 - float(p[1]))
        entropy = "%.3f" % round(float(confidence_from_probs(p, 2)), 3)
        rule = re.compile(r"%s reads %s[^.]*%s" % (re.escape(noul), re.escape(noul),
                                                   re.escape(entropy)))
        assert rule.search(text), (
            "the page must cite p(true)=%s as %s on a `noul` and %s as entropy; both come from "
            "confidence_from_probs above, so a number that drifts fails here" % (noul, noul,
                                                                                entropy))
    # the reason one sentence could not work: on a `noul` the shipped `confidence` *is* max(p), the
    # same quantity `answer_confidence` reports, which an entropy cannot be.
    for p_true in (0.60, 0.90):
        p = np.array([1.0 - p_true, p_true])
        assert close(max(float(p[1]), 1.0 - float(p[1])), answer_confidence(p, 2)), (
            "the page's `noul` answer would no longer carry the same number in both fields")


def _binning_targets(path):
    """What the installed binning map is applied to inside `_decode_answers`.

    The page closes its tour with "`binning_map` remaps `answer_confidence` and leaves `confidence`
    alone", which is a claim about the builder, not about prose: the map's input is the raw
    `answer_confidence`, and the two `confidence` expressions sit outside that branch.
    """
    src = _src03(path)
    func = next(n for n in ast.walk(ast.parse(src))
                if isinstance(n, ast.FunctionDef) and n.name == "_decode_answers")
    return [ast.get_source_segment(src, node.args[0]) for node in ast.walk(func)
            if (isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
                and node.func.id == "apply_binning_map" and node.args)]


def test_the_remap_reaches_only_answer_confidence():
    for path in (AGENT_PY, ONNX_PY):
        targets = _binning_targets(path)
        assert targets, "%s no longer applies a binning map inside the answer builder" % path
        assert all(t == "ans_raw" for t in targets), (
            "the page says a calibration payload remaps `answer_confidence` and leaves "
            "`confidence` alone, but %s remaps %s" % (path, sorted(set(targets))))


def test_the_two_agents_answer_the_same_way():
    """Parity arm: the ONNX path builds the same two expressions, so one page can be true of both."""
    agent_exprs = _confidence_exprs(AGENT_PY)
    onnx_exprs = _confidence_exprs(ONNX_PY)
    assert set(onnx_exprs) == {"choice", "score", "noul"}, (
        "the ONNX answer builder no longer yields all three types: %s" % sorted(onnx_exprs))
    assert "confidence_from_probs" in onnx_exprs["choice"], onnx_exprs["choice"]
    assert "confidence_from_probs" in onnx_exprs["score"], onnx_exprs["score"]
    assert onnx_exprs["noul"] == agent_exprs["noul"], (
        "the two agents now disagree on what a `noul` confidence is (%r vs %r), and the page "
        "describes only one of them" % (onnx_exprs["noul"], agent_exprs["noul"]))


for _fn in (test_page_drops_the_single_entropy_formula,
            test_page_names_the_formulas_the_agents_build,
            test_page_cites_the_numbers_the_repo_functions_produce,
            test_the_remap_reaches_only_answer_confidence,
            test_the_two_agents_answer_the_same_way):
    try:
        _fn()
    except AssertionError as e:
        FAIL.append("page-03/%s: %s" % (_fn.__name__, e))
    except Exception as e:                      # a crash is a failure, never a silent pass
        FAIL.append("page-03/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
    else:
        PASS.append("page-03/%s" % _fn.__name__)


for _fn in (test_preset_is_four_flags_and_one_rubric, test_helpers_are_live_and_pure,
            test_flag_line_prints_every_flag, test_past_half_uses_describes_cut,
            test_gaps_names_the_widest_separators, test_rank_cannot_assert_its_own_order,
            test_page_drops_the_hardcoded_conclusions,
            test_page_qualifies_the_banner_and_derives_its_numbers):
    try:
        _fn()
    except AssertionError as e:
        FAIL.append("page-28/%s: %s" % (_fn.__name__, e))
    except Exception as e:                      # a crash is a failure, never a silent pass
        FAIL.append("page-28/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
    else:
        PASS.append("page-28/%s" % _fn.__name__)


# --------------------------------------------- example 40 must not call a raw softmax calibrated
# `examples/40_caching_and_monitoring.py` gates 15 tickets on `answer_confidence`. On main it calls
# that field "the calibrated probability of the answer Laya reports" in both its banner and its
# closing paragraph, and its `bucket()` docstring reads "application policy over a calibrated
# number". `answer_confidence` is max(p) -- the quantity temperature scaling fits and ECE measures
# -- but the README's own Calibration section says that reading holds *only* after temperatures are
# fitted and validated, the shipped checkpoints are over-confident, and the loader emits at load
# time `RuntimeWarning: ... choice:11+=0.10058... -> 0.5. Treat confidence from the affected
# entries as uncalibrated.` The page now reads the temperature each shape was actually scaled by off
# the loaded agent and states the calibration as conditional.
#
# No weights are loaded here: the four helpers are pulled from the example's AST and exec'd against
# a fabricated agent, and `scale_for`'s lookup is checked against core's own `temp_bucket`, so the
# gate drives the code the page runs instead of re-reading its prose.
from laya.common import QTYPES as _QTYPES40, temp_bucket as _temp_bucket  # noqa: E402

EXAMPLE_40 = os.path.join(ROOT, "examples", "40_caching_and_monitoring.py")
HELPERS_40 = ("option_count", "scale_for", "clamped_buckets", "entropy_confidence")
# Names a helper may read from the page's world: the two core symbols it must defer to, and `math`
# for the entropy. Any other non-builtin free name means this gate cannot drive that helper.
CORE_GLOBALS_40 = {"temp_bucket", "QTYPES", "math"}

# main's page, as literal source lines. Every ban is witnessed against this text; every positive
# rule below is witnessed by showing this text does NOT satisfy it.
OLD_PAGE_40 = '''
    the triage preset and gates on `answer_confidence`, the calibrated probability of the
    """Our thresholds, not the model's: application policy over a calibrated number."""
   The monitor gates on `answer_confidence`: the calibrated probability of the answer Laya
   distribution has H/log(k) = %.2f, so `confidence` is %.2f while
   """ % (entropy, 1 - entropy, vague["intent"]["answer_confidence"]))
'''

UNCONDITIONAL_CALIBRATION = re.compile(r"the calibrated probability", re.I)
POLICY_OVER_CALIBRATED = re.compile(r"over a calibrated number", re.I)
SUBSTITUTED_CONFIDENCE = re.compile(r"1\s*-\s*entropy")
DERIVES_SCALING = re.compile(r"temp_bucket\(")
READS_REPORTED_CONFIDENCE = re.compile(r"\[[\"']confidence[\"']\]")
HARDCODED_BUCKET = re.compile(r"[\"'](choice|score|noul):[0-9]")


def _src40():
    with open(EXAMPLE_40, encoding="utf-8") as fh:
        return fh.read()


def _helpers40():
    """Exec only the example's top-level helpers -- its `load()` call needs weights."""
    tree = ast.parse(_src40())
    nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in HELPERS_40]
    ns = {"temp_bucket": _temp_bucket, "QTYPES": _QTYPES40, "math": math}
    exec(compile(ast.Module(body=nodes, type_ignores=[]), EXAMPLE_40, "exec"), ns)  # noqa: S102
    return ns


def _called40():
    tree = ast.parse(_src40())
    top = [n for n in tree.body if not isinstance(
        n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Import, ast.ImportFrom))]
    return {node.id for stmt in top for node in ast.walk(stmt)
            if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)}


def _fn40(name):
    for n in ast.parse(_src40()).body:
        if isinstance(n, ast.FunctionDef) and n.name == name:
            return n
    raise KeyError(name)


class _FakeAgent(object):
    """Only the three temperature attributes `scale_for` and `clamped_buckets` read."""

    def __init__(self, by_options, per_type, raw=None):
        self.temperature_by_options = by_options
        self.temperature = per_type
        self.temperature_by_options_raw = dict(by_options) if raw is None else raw


def test_page40_helpers_are_live_defer_to_core():
    defined = sorted(n.name for n in ast.parse(_src40()).body
                     if isinstance(n, ast.FunctionDef) and n.name in HELPERS_40)
    assert defined == sorted(HELPERS_40), "example 40 lost a helper: %s" % defined
    ns = _helpers40()
    unused = set(HELPERS_40) - _called40()
    assert not unused, "defined but never called at module level: %s" % sorted(unused)
    for name in HELPERS_40:
        node = _fn40(name)
        outside = sorted(n for n in _free_names(node)
                         if not hasattr(builtins, n) and n not in CORE_GLOBALS_40)
        assert not outside, "%s reads %s from the page, so this gate cannot drive it" % (name, outside)
        prints = [c for c in ast.walk(node)
                  if isinstance(c, ast.Call) and isinstance(c.func, ast.Name) and c.func.id == "print"]
        assert not prints, "%s prints instead of returning" % name
    # `scale_for` must resolve the bucket through core, not carry a typed-in table.
    assert not HARDCODED_BUCKET.search(ast.dump(_fn40("scale_for"))), (
        "scale_for hardcodes a bucket name instead of calling temp_bucket")


def test_scale_for_replays_core_lookup():
    """For every shape, scale_for returns exactly `temperature_by_options.get(bucket, per_type)`."""
    ns = _helpers40()
    per_type = [7.0, 8.0, 9.0]                       # choice, score, noul fallbacks
    by_options = {"choice:6-10": 1.0, "noul:2": 1.9834, "score:3-5": 1.2514}
    fake = _FakeAgent(by_options, per_type)
    for qtype, k in (("choice", 6), ("noul", 2), ("score", 4), ("noul", 2)):
        name, applied, in_map = ns["scale_for"](fake, qtype, k)
        bucket = _temp_bucket(_QTYPES40[qtype], k)
        assert name == bucket, (qtype, k, name, bucket)
        assert in_map == (bucket in by_options), (bucket, in_map)
        assert abs(applied - by_options.get(bucket, per_type[_QTYPES40[qtype]])) < 1e-9, (
            "%s: %r not core's get(...)" % (bucket, applied))
    # the fallback arm: an unmapped shape must fall to the per-type temperature, flagged not-in-map.
    name, applied, in_map = ns["scale_for"](_FakeAgent({}, [0.5, 0.6, 0.7]), "choice", 12)
    assert (name, applied, in_map) == ("choice:11+", 0.5, False), (name, applied, in_map)
    # a mapped temperature of exactly 1.0 must be reported as 1.0, not nudged.
    assert ns["scale_for"](_FakeAgent({"choice:6-10": 1.0}, [1.0, 1.0, 1.0]), "choice", 6) == (
        "choice:6-10", 1.0, True)


def test_option_count_matches_the_shipped_preset():
    ns = _helpers40()
    counts = {qid: ns["option_count"](q) for qid, q in laya.triage_questions().items()}
    assert counts == {"intent": 6, "is_urgent": 2, "frustration": 4,
                      "refund_requested": 2, "churn_risk": 2}, counts


def test_clamped_buckets_names_only_a_shipped_value_the_runtime_refused():
    ns = _helpers40()
    # `choice:11+` ships below TEMP_MIN and is clamped to 0.5; everything else ships unchanged.
    applied = {"choice:11+": 0.5, "noul:2": 1.9834}
    raw = {"choice:11+": 0.10058280825614929, "noul:2": 1.9834}
    out = ns["clamped_buckets"](_FakeAgent(applied, [0.5, 0.5, 0.5], raw))
    assert out == ["choice:11+=0.1006 -> 0.5000"], out
    # and an honest checkpoint reports nothing.
    same = {"noul:2": 1.9834}
    assert ns["clamped_buckets"](_FakeAgent(same, [0.5, 0.5, 0.5], dict(same))) == []


def test_entropy_confidence_recomputes_core_exactly():
    ns = _helpers40()
    for dist in ([0.5, 0.3, 0.2], [0.9, 0.1], [0.25, 0.25, 0.25, 0.25], [1.0, 0.0, 0.0]):
        p = np.array(dist, dtype=float)
        got = ns["entropy_confidence"](dist)
        want = confidence_from_probs(p, len(dist))
        assert close(got, want, 1e-9), (dist, got, want)
    assert ns["entropy_confidence"]([1.0]) == 1.0, "k<2 is 1.0 by definition"


def test_page40_drops_the_unconditional_calibration_claim():
    """Each ban fires on main's page and not on this one -- a ban with no witness is a guess."""
    src = _src40()
    for name, rule in (("'the calibrated probability'", UNCONDITIONAL_CALIBRATION),
                       ("'over a calibrated number'", POLICY_OVER_CALIBRATED),
                       ("prints `1 - entropy` as the confidence", SUBSTITUTED_CONFIDENCE)):
        assert rule.search(OLD_PAGE_40), "%s does not fire on the wording it bans" % name
        assert not rule.search(src), "%s is still in example 40" % name


def test_page40_reads_the_scaling_and_the_reported_field():
    """The positive half: the page must derive the temperature and print the reported field."""
    src = _src40()
    assert DERIVES_SCALING.search(src), "the bucket must be resolved through core's temp_bucket"
    assert READS_REPORTED_CONFIDENCE.search(src), "the page must read the reported `confidence`"
    # and main's page satisfies none of that, so these rules could not have passed before.
    assert not DERIVES_SCALING.search(OLD_PAGE_40)
    assert not READS_REPORTED_CONFIDENCE.search(OLD_PAGE_40)


for _fn in (test_page40_helpers_are_live_defer_to_core, test_scale_for_replays_core_lookup,
            test_option_count_matches_the_shipped_preset,
            test_clamped_buckets_names_only_a_shipped_value_the_runtime_refused,
            test_entropy_confidence_recomputes_core_exactly,
            test_page40_drops_the_unconditional_calibration_claim,
            test_page40_reads_the_scaling_and_the_reported_field):
    try:
        _fn()
    except AssertionError as e:
        FAIL.append("page-40/%s: %s" % (_fn.__name__, e))
    except Exception as e:                      # a crash is a failure, never a silent pass
        FAIL.append("page-40/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
    else:
        PASS.append("page-40/%s" % _fn.__name__)
# Gate: the two predict_long call sites' comments must describe the None-min_confidence
# contract truthfully. The pre-fix wording claimed every answer "reports that it ran ungated"
# / "say so on every answer"; confidence.py:196-197 documents and :211-212 implements the
# opposite -- the gate writes nothing at all, so the payload carries no `abstention` field.
import ast as _gate_ast  # noqa: E402

_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


def _gate_gate_comment(rel_path):
    """Return the run of comment lines that immediately precede the
    apply_confidence_gate([result], None) call inside predict_long."""
    with open(os.path.join(_REPO, rel_path)) as f:
        lines = f.readlines()
    tree = _gate_ast.parse("".join(lines))
    for node in _gate_ast.walk(tree):
        if isinstance(node, _gate_ast.FunctionDef) and node.name == "predict_long":
            call_line = None
            for sub in _gate_ast.walk(node):
                if (isinstance(sub, _gate_ast.Call)
                        and getattr(sub.func, "id", getattr(sub.func, "attr", None)) == "apply_confidence_gate"
                        and len(sub.args) == 2
                        and isinstance(sub.args[1], _gate_ast.Constant)
                        and sub.args[1].value is None):
                    call_line = sub.lineno - 1
                    break
            if call_line is None:
                continue
            out = []
            i = call_line - 1
            while i >= 0 and lines[i].lstrip().startswith("#"):
                out.insert(0, lines[i].strip().lstrip("#").strip())
                i -= 1
            return " ".join(out)
    return ""


_agent_comment = _gate_gate_comment(os.path.join("laya", "agent.py"))
_onnx_comment = _gate_gate_comment(os.path.join("laya", "onnx_agent.py"))

for _label, _comment in (("agent.py", _agent_comment), ("onnx_agent.py", _onnx_comment)):
    check_true("gate/comment/%s exists" % _label, len(_comment) > 0, "no comment found")
    check_true("gate/comment/%s drops the ungated-reports claim" % _label,
               "reports that it ran ungated" not in _comment
               and "say so on every answer" not in _comment,
               "pre-fix wording still on the call site: %r" % _comment)
    check_true("gate/comment/%s names writes nothing" % _label,
               "writes nothing" in _comment,
               "gate comment must name the None-contract: %r" % _comment)
    check_true("gate/comment/%s names the min_confidence argument" % _label,
               "min_confidence" in _comment,
               "gate comment must refer to the argument it is about: %r" % _comment)

# Live witness: apply_confidence_gate([payload], None) leaves the payload byte-identical,
# so the corrected comment is not itself a claim the code contradicts.
_witness = {"model": "w", "answers": {"q": {"choice": "a", "confidence": 0.42}}, "usage": {}}
_witness_before = copy.deepcopy(_witness)
apply_confidence_gate([_witness], None)
check("gate/live None writes nothing", _witness, _witness_before)
check("gate/live None adds no abstention key", "abstention" in _witness["answers"]["q"], False)


# --------------------------------------------- example 18 must not call the threshold a measurement
# `examples/18_confidence_gating.py` is the intro-level gating pattern: it picks a 0.85 cutoff on
# `answer_confidence` and acts on it. On main it calls the field "the calibrated probability Laya
# puts on the answer it reports" and its module docstring says it "branches on Laya's calibrated
# confidence". The README's Calibration section (line 1586) says the opposite about the shipped
# state: mean ECE on `laya` is 0.466 as shipped, dropping to 0.081 only after refitting one
# temperature per (question type, option-count) bucket on held-out data, and both shipped
# checkpoints are over-confident. `answer_confidence` is the field temperature scaling fits and
# the abstention gate reads -- but "calibrated" is a property a fit earns, not a property of the
# field, and the loader emits `RuntimeWarning: ... Treat confidence from the affected entries as
# uncalibrated.` on this very checkpoint. The page now reads back the temperature it actually
# applies to each of its two questions and states the calibration claim as conditional.
from laya.common import QTYPES as _QTYPES18, temp_bucket as _temp_bucket18  # noqa: E402

EXAMPLE_18 = os.path.join(ROOT, "examples", "18_confidence_gating.py")
HELPERS_18 = ("option_count", "scale_for")
CORE_GLOBALS_18 = {"temp_bucket", "QTYPES", "math"}

OLD_PAGE_18 = '''
Answers a batch of support emails and branches on Laya\'s calibrated confidence: act
    The production pattern from the README. Gate on `answer_confidence`: the calibrated
    probability Laya puts on the answer it reports, defined the same way on every question
'''

UNQUALIFIED_CALIBRATION_18 = re.compile(
    r"the\s+calibrated\s+probability|Laya['’]s\s+calibrated\s+confidence", re.I)
DERIVES_SCALING_18 = re.compile(r"temp_bucket\(")
NAMES_CONDITION_18 = re.compile(r"over-confident", re.I)
HARDCODED_BUCKET_18 = re.compile(r"[\"'](choice|score|noul):[0-9]")


def _src18():
    with open(EXAMPLE_18, encoding="utf-8") as fh:
        return fh.read()


def _helpers18():
    tree = ast.parse(_src18())
    nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in HELPERS_18]
    ns = {"temp_bucket": _temp_bucket18, "QTYPES": _QTYPES18, "math": math}
    exec(compile(ast.Module(body=nodes, type_ignores=[]), EXAMPLE_18, "exec"), ns)  # noqa: S102
    return ns


class _FakeAgent18(object):
    """Only the two temperature attributes `scale_for` reads."""

    def __init__(self, by_options, per_type):
        self.temperature_by_options = by_options
        self.temperature = per_type


def _top_literal18(name):
    for n in ast.parse(_src18()).body:
        if isinstance(n, ast.Assign) and any(
                isinstance(t, ast.Name) and t.id == name for t in n.targets):
            return ast.literal_eval(n.value)
    raise KeyError(name)


def _fn18(name):
    for n in ast.parse(_src18()).body:
        if isinstance(n, ast.FunctionDef) and n.name == name:
            return n
    raise KeyError(name)


def test_page18_helpers_are_live_and_defer_to_core():
    defined = sorted(n.name for n in ast.parse(_src18()).body
                     if isinstance(n, ast.FunctionDef) and n.name in HELPERS_18)
    assert defined == sorted(HELPERS_18), "example 18 lost a helper: %s" % defined
    tree = ast.parse(_src18())
    top_loads = {node.id for stmt in tree.body if not isinstance(
        stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Import, ast.ImportFrom))
        for node in ast.walk(stmt)
        if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)}
    unused = set(HELPERS_18) - top_loads
    assert not unused, "defined but never called at module level: %s" % sorted(unused)
    for name in HELPERS_18:
        node = _fn18(name)
        outside = sorted(n for n in _free_names(node)
                         if not hasattr(builtins, n) and n not in CORE_GLOBALS_18)
        assert not outside, "%s reads %s from the page" % (name, outside)
        prints = [c for c in ast.walk(node)
                  if isinstance(c, ast.Call) and isinstance(c.func, ast.Name) and c.func.id == "print"]
        assert not prints, "%s prints instead of returning" % name
    assert not HARDCODED_BUCKET_18.search(ast.dump(_fn18("scale_for"))), (
        "scale_for hardcodes a bucket name instead of calling temp_bucket")


def test_page18_questions_reach_core_bucket_names():
    """Replay the page's own QUESTIONS through core's temp_bucket and through the example's helper."""
    ns = _helpers18()
    questions = _top_literal18("QUESTIONS")
    per_type = [7.0, 8.0, 9.0]
    by_options = {"choice:3-5": 1.7601518630981445, "noul:2": 1.983399510383606}
    fake = _FakeAgent18(by_options, per_type)
    shapes = {}
    for qid, q in questions.items():
        k = ns["option_count"](q)
        name, applied, in_map = ns["scale_for"](fake, q["type"], k)
        assert name == _temp_bucket18(_QTYPES18[q["type"]], k), (qid, name)
        assert abs(applied - by_options[name]) < 1e-9, (qid, applied)
        assert in_map is True, "%s: %s missing from map" % (qid, name)
        shapes[qid] = (q["type"], k, name, applied)
    assert shapes == {
        "department": ("choice", 4, "choice:3-5", 1.7601518630981445),
        "refund_requested": ("noul", 2, "noul:2", 1.983399510383606),
    }, shapes


def test_page18_drops_the_unconditional_calibration_claim():
    """The two phrases main uses fire on the ban and are gone here; main has no temp_bucket call."""
    src = _src18()
    assert UNQUALIFIED_CALIBRATION_18.search(OLD_PAGE_18), "the ban must fire on main's wording"
    assert not UNQUALIFIED_CALIBRATION_18.search(src), (
        "the page still calls `answer_confidence` calibrated without condition")
    assert not DERIVES_SCALING_18.search(OLD_PAGE_18), "main's page has no scaling lookup"
    assert not NAMES_CONDITION_18.search(OLD_PAGE_18), "main's page states no calibration condition"
    assert DERIVES_SCALING_18.search(src), "the page must resolve the bucket via core's temp_bucket"
    assert NAMES_CONDITION_18.search(src), (
        "the page must carry the README's Calibration condition (mean ECE 0.466 -> 0.081, "
        "both shipped checkpoints over-confident)")
    # The README's exact language lives at line 1586; the page must not misquote it.
    readme = open(os.path.join(ROOT, "README.md"), encoding="utf-8").read()
    assert "Both checkpoints are over-confident as shipped" in readme
    assert "0.466 -> 0.081" in readme


for _fn in (test_page18_helpers_are_live_and_defer_to_core,
            test_page18_questions_reach_core_bucket_names,
            test_page18_drops_the_unconditional_calibration_claim):
    try:
        _fn()
    except AssertionError as e:
        FAIL.append("page-18/%s: %s" % (_fn.__name__, e))
    except Exception as e:
        FAIL.append("page-18/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
    else:
        PASS.append("page-18/%s" % _fn.__name__)

# ------------------------------------------ example 30's score confidence recomputation
# `examples/30_custom_schema_design.py` teaches the `score` answer type on the page. Main's
# version asserted -- in prose, with nothing to check it against -- that "`confidence` on a
# score is normalised entropy, so a wide-but-ordered distribution looks unconfident". That is
# the inverse of the repo's own definition: `laya.common.confidence_from_probs` returns
# `1 - H(p) / log(k)`, examples 18/28/40 all describe it as "1 minus normalised entropy", and
# the sentence was self-contradictory in place -- if confidence were the entropy itself, a wide
# distribution would read as HIGH confidence, not "unconfident".
#
# The fix rewrites the sentence to the correct definition AND recomputes the field on the spot:
# `entropy_confidence(list(probabilities.values()))` is printed next to
# `answer["severity"]["confidence"]` for both records, so the reader sees 1 - H/log(k) produce
# the reported 0.2314 / 0.2019 to within rounding. No weights are loaded here; the gate execs
# the helper from the example's own AST and drives it against `confidence_from_probs`.
EXAMPLE_30 = os.path.join(ROOT, "examples", "30_custom_schema_design.py")
HELPERS_30 = ("entropy_confidence",)

# The page as it ships on main: the sentence this PR replaces, verbatim. The ban below must
# fire on this text and not on the current example; the positive rules must fire on the
# current example and not on this text.
OLD_PAGE_30 = '''
print("   them, and a choice when the labels are unordered. Read the score itself: `confidence`")
print("   on a score is normalised entropy, so a wide-but-ordered distribution looks")
print("   unconfident even when the expected level is informative.")
'''

# The definition stated as the entropy itself rather than 1 - H/log(k).
INVERTED_SCORE_DEF = re.compile(
    r"`confidence`[^.]{0,120}?\bon a score is (?:the )?normali[sz]ed entropy", re.I | re.S)
# The correct shape, either symbolic or spelled out.
SCORE_DEFINES_1_MINUS_H = re.compile(
    r"1\s*-\s*H\s*/\s*log\(k\)|one\s+minus\s+the\s+normali[sz]ed\s+entropy", re.I)
# The recomputation call must be in the source, not just asserted. Two parts: the page must
# read the reported probabilities off the answer, and it must feed them into the helper.
PROBS_READ = re.compile(
    r"list\(\s*ans\[\"severity\"\]\[\"probabilities\"\]\s*\.\s*values\(\)\s*\)")
HELPER_CALLED = re.compile(r"entropy_confidence\(")
# Reported and recomputed values printed side by side (either literal digits or a %.Nf
# conversion the page formats them with).
PRINTS_REPORTED_AND_RECOMPUTED = re.compile(
    r"reported\s+(?:%?\.\d+f|[\d.]+)\s*,\s*recomputed", re.I)


def _src30():
    with open(EXAMPLE_30, encoding="utf-8") as fh:
        return fh.read()


def _helpers30():
    """Exec the example's pure top-level helpers. `entropy_confidence` reads `math`."""
    tree = ast.parse(_src30())
    nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in HELPERS_30]
    ns = {"math": math}
    exec(compile(ast.Module(body=nodes, type_ignores=[]), EXAMPLE_30, "exec"), ns)  # noqa: S102
    return ns, {n.name for n in nodes}


def _called30():
    """Names the example's top-level *statements* use, so a helper cannot pass by being dead."""
    tree = ast.parse(_src30())
    top = [n for n in tree.body if not isinstance(
        n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Import, ast.ImportFrom))]
    return {node.id for stmt in top for node in ast.walk(stmt)
            if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)}


def test_page30_helpers_are_live_and_pure():
    """`entropy_confidence` is defined, called at top level, and returns rather than prints."""
    _, defined = _helpers30()
    assert defined == set(HELPERS_30), "example 30 lost a helper: %s" % sorted(defined)
    unused = set(HELPERS_30) - _called30()
    assert not unused, "defined but never called at module level: %s" % sorted(unused)
    tree = ast.parse(_src30())
    for name in HELPERS_30:
        node = next(n for n in tree.body
                    if isinstance(n, ast.FunctionDef) and n.name == name)
        prints = [c for c in ast.walk(node)
                  if isinstance(c, ast.Call) and isinstance(c.func, ast.Name)
                  and c.func.id == "print"]
        assert not prints, "%s prints instead of returning" % name


def test_entropy_confidence_matches_core():
    """Recomputing 1 - H/log(k) by hand must equal `confidence_from_probs` on the same list."""
    ns = _helpers30()[0]
    for probs in ([0.5, 0.5], [0.1, 0.9], [0.25, 0.25, 0.25, 0.25],
                  [0.7, 0.2, 0.05, 0.05], [1e-9, 0.999999999, 0.0],
                  [0.4, 0.3, 0.2, 0.1], [0.05, 0.15, 0.3, 0.35, 0.1, 0.05]):
        got = ns["entropy_confidence"](list(probs))
        want = confidence_from_probs(np.array(probs, dtype=float), len(probs))
        assert abs(got - want) < 1e-9, (probs, got, want)
    assert ns["entropy_confidence"]([1.0]) == 1.0, "k=1 is certain"
    assert ns["entropy_confidence"]([]) == 1.0, "k=0 does not crash"


def test_page30_drops_the_inverted_definition():
    """The ban fires on main's sentence and not on this one; the positive rule inverts."""
    src = _src30()
    assert INVERTED_SCORE_DEF.search(OLD_PAGE_30), \
        "the ban does not fire on main's wording"
    assert not INVERTED_SCORE_DEF.search(src), \
        "the inverted definition is still in example 30"
    assert SCORE_DEFINES_1_MINUS_H.search(src), \
        "the correct 1 - H/log(k) definition is missing"
    assert not SCORE_DEFINES_1_MINUS_H.search(OLD_PAGE_30), \
        "the positive rule passes on main's page too"


def test_page30_recomputes_in_place():
    """The reader sees the field cross-checked against the probabilities that produced it."""
    src = _src30()
    assert PROBS_READ.search(src), \
        "the score section no longer reads the reported `probabilities` list"
    assert HELPER_CALLED.search(src), \
        "the score section no longer calls `entropy_confidence`"
    assert PRINTS_REPORTED_AND_RECOMPUTED.search(src), \
        "reported and recomputed values are not printed side by side"
    assert not PROBS_READ.search(OLD_PAGE_30)
    assert not HELPER_CALLED.search(OLD_PAGE_30)
    assert not PRINTS_REPORTED_AND_RECOMPUTED.search(OLD_PAGE_30)


for _fn in (test_page30_helpers_are_live_and_pure,
            test_entropy_confidence_matches_core,
            test_page30_drops_the_inverted_definition,
            test_page30_recomputes_in_place):
    try:
        _fn()
    except AssertionError as e:
        FAIL.append("page-30/%s: %s" % (_fn.__name__, e))
    except Exception as e:
        FAIL.append("page-30/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
    else:
        PASS.append("page-30/%s" % _fn.__name__)


# -------------------------------- evals' --min-confidence wording must match the gate's effect
#
# `laya/evals_cli.py`, `laya/evals.py`, `docs/evals.md`, `tests/test_evals.py` and
# `tests/test_evals_api.py` each describe the abstention gate's role in a scored run. Their
# previous wording claimed the threshold changes what scores: "answers below it come back
# abstained", "an abstention overwrites a low-confidence choice", "the run scores the policy
# at that threshold, not the raw argmax", "a `precision@coverage` figure for a policy that
# never ran". But `apply_confidence_gate` in this module writes `low_confidence` and
# `abstention` state fields and leaves `answer["choice"]/["noul"]/["score"]` untouched, and
# every `Evaluator.score` and `_correct` in `laya/evals.py` reads only those value keys. So the
# metrics -- accuracy, ece, brier, aurc, selective_accuracy@NN -- are identical at every
# threshold; what changes is `report.config["timing"]["min_confidence"]` and `min_confidence_sent`,
# which the wording below must describe as claims about the run, not claims about the answers.

_EVALS_CLI_PATH = os.path.join(ROOT, "laya", "evals_cli.py")
_EVALS_PY_PATH = os.path.join(ROOT, "laya", "evals.py")
_EVALS_MD_PATH = os.path.join(ROOT, "docs", "evals.md")
_TEST_EVALS_PATH = os.path.join(ROOT, "tests", "test_evals.py")
_TEST_EVALS_API_PATH = os.path.join(ROOT, "tests", "test_evals_api.py")


def _read_text(path):
    with open(path, "r", encoding="utf-8") as f:
        return f.read()


def _docstring_of(path, func_name):
    """A function's docstring pulled out of the file, so a ban can point at one site not the file."""
    tree = ast.parse(_read_text(path))
    for node in ast.walk(tree):
        if isinstance(node, ast.FunctionDef) and node.name == func_name:
            return ast.get_docstring(node) or ""
    return ""


def _cli_help_of(path, flag):
    """The concatenated `help=` literal on a `parser.add_argument(flag, ...)` call."""
    tree = ast.parse(_read_text(path))
    for node in ast.walk(tree):
        if isinstance(node, ast.Call) and node.args \
                and isinstance(node.args[0], ast.Constant) and node.args[0].value == flag:
            for kw in node.keywords:
                if kw.arg == "help":
                    try:
                        return ast.literal_eval(kw.value) or ""
                    except (ValueError, SyntaxError):
                        return ""
    return ""


# The wrong-language patterns: each is a claim that the threshold changes the scored numbers.
_BAN_COME_BACK_ABSTAINED = re.compile(r"come back\s+abstained", re.I)
_BAN_OVERWRITES_A_LOW = re.compile(r"abstention\s+overwrites\s+a\s+low", re.I)
_BAN_SCORES_POLICY = re.compile(r"scores the policy at (?:the|that) threshold", re.I)
_BAN_VS_RAW_ARGMAX = re.compile(r"(?:rather than|not)\s+the\s+raw\s+argmax", re.I)
_BAN_MINCONFIDENCE_CHANGES_ANSWER = re.compile(
    r"`min_confidence`[^.\n]{0,80}?changes the answ", re.I)
_BAN_UNLIKE_GROUPING_CHANGES = re.compile(
    r"[Uu]nlike\s+`?grouping`?[^.]{0,40}?changes the answ", re.I)
_BAN_PRECISION_NUMBER_FOR_POLICY = re.compile(
    r"precision@coverage`?\s+(?:number|figure)\s+for a policy that never ran", re.I)
_BAN_WOULD_REPORT_PRECISION = re.compile(
    r"would report `?precision@coverage`?", re.I)
_BAN_OTHERWISE_SCORE_POLICY = re.compile(
    r"otherwise score a policy that never ran", re.I)
_BAN_NO_WAY_TO_MEASURE_PATCOV = re.compile(
    r"no way to measure `?precision@coverage", re.I)
_BAN_A_SCORING_CONTROL = re.compile(r"(?:it is|that is|dropping) a\b[^.\n]{0,10}scoring control", re.I)
_BAN_ANSWER_NOT_SAME_DECISION = re.compile(
    r"an answer below the threshold is not the same decision", re.I)
_BAN_T0_T07_DIFFERENT_EXPERIMENT = re.compile(
    r"same run at `T=0` and `T=0\.7` is a different experiment", re.I)
_BAN_SWEEP_IS_SERIES = re.compile(
    r"`precision@coverage`\s+sweep is a series of these", re.I)
_BAN_CHANGES_WHICH_ANSWERS_SCORE = re.compile(
    r"(?:abstention threshold|min_confidence).{0,30}?changes which answers score", re.I | re.S)

_EVALS_BANS = (
    _BAN_COME_BACK_ABSTAINED, _BAN_OVERWRITES_A_LOW, _BAN_SCORES_POLICY,
    _BAN_VS_RAW_ARGMAX, _BAN_MINCONFIDENCE_CHANGES_ANSWER,
    _BAN_UNLIKE_GROUPING_CHANGES, _BAN_PRECISION_NUMBER_FOR_POLICY,
    _BAN_WOULD_REPORT_PRECISION, _BAN_OTHERWISE_SCORE_POLICY,
    _BAN_NO_WAY_TO_MEASURE_PATCOV, _BAN_A_SCORING_CONTROL,
    _BAN_ANSWER_NOT_SAME_DECISION, _BAN_T0_T07_DIFFERENT_EXPERIMENT,
    _BAN_SWEEP_IS_SERIES, _BAN_CHANGES_WHICH_ANSWERS_SCORE,
)

# Positive claims the wording must make: name the two state fields `apply_confidence_gate`
# writes. A wording that only says what the gate does NOT do is a diff, not a fix.
_SAYS_LOW_CONFIDENCE = re.compile(r"low_confidence")
_SAYS_ABSTENTION_FIELD = re.compile(
    r"abstention[^a-zA-Z]{0,3}[:=]?[^a-zA-Z]{0,3}[\"']abstained[^a-zA-Z]{0,3}[\"']")

# The pre-fix wording at each of the ten sites we touched, so every ban above has a witness
# it fires on. A ban that never fires is a rule the code cannot check.
_OLD_E1 = ("abstention threshold on `answer_confidence` (#361): answers below it come back "
           "abstained, so the run scores the policy at that threshold rather than the raw "
           "argmax.")
_OLD_E2 = ("`min_confidence` changes the answer (an abstention overwrites a low-confidence "
           "choice), so it is a scoring control, not an optimisation. silently dropping the "
           "threshold and reporting the same run would give a `precision@coverage` number for "
           "a policy that never ran")
_OLD_E3 = ("answers below it come back abstained, so the run scores the policy at that "
           "threshold, not the raw argmax. Unlike `sort_by_length` this changes the answers")
_OLD_E4 = ("because an abstention threshold changes which answers score as correct. Silently "
           "dropping it would publish a `precision@coverage` number for a policy that never ran")
_OLD_E5 = ("or drop the threshold -- the report would otherwise score a policy that never ran")
_OLD_MD1 = ("Unlike grouping, this changes the answers that score: the same run at `T=0` and "
            "`T=0.7` is a different experiment, and a `precision@coverage` sweep is a series "
            "of these, not a single baseline drifting.")
_OLD_MD2 = ("Silently dropping a scoring control is the class of lie this harness exists to "
            "prevent: the report would publish a `precision@coverage` figure for a policy "
            "that never ran.")
_OLD_T1 = ("That is a scoring control: an answer below the threshold is not the same decision "
           "as one above. A `laya-evals run` that could not pass it through had no way to "
           "measure `precision@coverage` at any threshold")
_OLD_T2 = ("Silently dropping a scoring control would report `precision@coverage` for a "
           "policy that never ran.")
_OLD_T3 = ("a `--min-confidence` run that silently dropped the argument would publish a "
           "`precision@coverage` figure for a policy that never ran")

_OLD_EVALS_SITES = (_OLD_E1, _OLD_E2, _OLD_E3, _OLD_E4, _OLD_E5,
                    _OLD_MD1, _OLD_MD2, _OLD_T1, _OLD_T2, _OLD_T3)


def test_abstention_wording_bans_have_teeth():
    """Every ban fires on a pre-fix site, and every pre-fix site is caught by at least one ban."""
    matched = [[bool(rule.search(old)) for rule in _EVALS_BANS] for old in _OLD_EVALS_SITES]
    for i, old in enumerate(_OLD_EVALS_SITES):
        assert any(matched[i]), "site %d's pre-fix wording is not caught by any ban:\n%s" % (i, old)
    for j, rule in enumerate(_EVALS_BANS):
        assert any(matched[i][j] for i in range(len(_OLD_EVALS_SITES))), \
            "ban /%s/ never fires on any pre-fix wording: a rule without a witness" % rule.pattern


def test_abstention_wording_is_gone_from_every_site():
    """None of the scoring-change claims may appear in the sources they were removed from."""
    for path in (_EVALS_CLI_PATH, _EVALS_PY_PATH, _EVALS_MD_PATH,
                 _TEST_EVALS_PATH, _TEST_EVALS_API_PATH):
        src = _read_text(path)
        for rule in _EVALS_BANS:
            assert not rule.search(src), \
                "%s still carries the scoring-change claim: /%s/" % (path, rule.pattern)


def test_abstention_wording_names_the_state_fields():
    """Each site individually must name what the gate writes, not just somewhere in the file.

    Per-docstring rather than per-file, so a mutation that drops `low_confidence` from the
    `evaluate()` docstring alone is caught even though the sibling `_takes_min_confidence`
    docstring above it still names the field.
    """
    help_text = _cli_help_of(_EVALS_CLI_PATH, "--min-confidence")
    assert help_text, "no `help=` string on the --min-confidence argument in laya/evals_cli.py"
    assert _SAYS_LOW_CONFIDENCE.search(help_text), \
        "evals_cli's --min-confidence help does not name the `low_confidence` field"
    assert _SAYS_ABSTENTION_FIELD.search(help_text), \
        "evals_cli's --min-confidence help does not name `abstention: \"abstained\"`"

    for func_name in ("_takes_min_confidence", "evaluate"):
        doc = _docstring_of(_EVALS_PY_PATH, func_name)
        assert doc, "no docstring on laya/evals.py::%s" % func_name
        assert _SAYS_LOW_CONFIDENCE.search(doc), \
            "laya/evals.py::%s docstring does not name the `low_confidence` field" % func_name
        assert _SAYS_ABSTENTION_FIELD.search(doc), \
            "laya/evals.py::%s docstring does not name `abstention: \"abstained\"`" % func_name

    md = _read_text(_EVALS_MD_PATH)
    assert _SAYS_LOW_CONFIDENCE.search(md), \
        "docs/evals.md does not name the `low_confidence` field"
    assert _SAYS_ABSTENTION_FIELD.search(md), \
        "docs/evals.md does not name `abstention: \"abstained\"`"


def test_gate_writes_the_fields_the_docs_name():
    """The witness that ties the wording above to `apply_confidence_gate`'s actual effect.

    Drive the gate at 0.7 over one high-confidence and one low-confidence `choice` answer, both
    carrying `choice: "keep"`. The low one must come back flagged; both must keep their raw
    argmax in `answer["choice"]`. If the gate ever starts overwriting the value, this assertion
    breaks and the wording claim breaks with it.
    """
    results = [{"answers": {
        "q_high": {"type": "choice", "choice": "keep",
                    "answer_confidence": 0.90,
                    "probabilities": {"keep": 0.90, "drop": 0.10}},
        "q_low": {"type": "choice", "choice": "keep",
                   "answer_confidence": 0.40,
                   "probabilities": {"keep": 0.40, "drop": 0.60}},
    }}]
    apply_confidence_gate(results, 0.7)
    high = results[0]["answers"]["q_high"]
    low = results[0]["answers"]["q_low"]
    check("evals-wording/high clears the gate", high["abstention"], GATE_PASSED)
    check("evals-wording/low falls below", low["abstention"], GATE_ABSTAINED)
    check_true("evals-wording/low is flagged", low.get("low_confidence") is True)
    check_true("evals-wording/high is not flagged", "low_confidence" not in high)
    # The claim the wording makes about the gate: the value key stays at the raw argmax.
    check("evals-wording/high's choice is unchanged", high["choice"], "keep")
    check("evals-wording/low's choice is unchanged even when abstained", low["choice"], "keep")
    # Both thresholds echoed onto the answer, so `abstention_threshold` matches the wording.
    check("evals-wording/high echoes the threshold", high["abstention_threshold"], 0.7)
    check("evals-wording/low echoes the threshold", low["abstention_threshold"], 0.7)


for _fn in (test_abstention_wording_bans_have_teeth,
            test_abstention_wording_is_gone_from_every_site,
            test_abstention_wording_names_the_state_fields,
            test_gate_writes_the_fields_the_docs_name):
    try:
        _fn()
    except AssertionError as e:
        FAIL.append("evals-abstention/%s: %s" % (_fn.__name__, e))
    except Exception as e:
        FAIL.append("evals-abstention/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
    else:
        PASS.append("evals-abstention/%s" % _fn.__name__)

# ------------------------------------------ example 26's banner must count the primitives triage_questions() returns
# `examples/26_presets_triage.py` opened with "`triage_questions()` is a ready-made schema for
# inbound support: five questions ... a multi-way choice, two yes/no probabilities, and an ordinal
# score". `laya.triage_questions()` ships three `noul` fields -- `is_urgent`, `refund_requested`,
# `churn_risk` -- plus the choice and the score, so the banner's own arithmetic read 1 + 2 + 1 = 4
# against its stated "five questions", and the example's own print block already prints all three
# `noul` values two paragraphs later. A reader counting primitives against the schema could not
# reconcile them.
EXAMPLE_26 = os.path.join(ROOT, "examples", "26_presets_triage.py")
OLD_BANNER_26 = '''
    `triage_questions()` is a ready-made schema for inbound support: five questions, one
    forward pass. It mixes all three primitives -- a multi-way choice, two yes/no
    probabilities, and an ordinal score -- so a single call fills a whole triage record.
'''
_BAN_TWO_YESNO = re.compile(r"\btwo\s+yes/no\s+probabilit", re.I)
# The banner's own stated counts. `_WORDS` maps each spelling that could stand for one primitive.
_STATED_TOTAL_26 = re.compile(r"\b(one|two|three|four|five|six)\s+questions", re.I)
_STATED_YESNO_26 = re.compile(r"\b(one|two|three|four|five|six)\s+yes/no\s+probabilit", re.I)
_STATED_CHOICE_26 = re.compile(r"\b(a|one)\s+multi-way\s+choice", re.I)
_STATED_SCORE_26 = re.compile(r"\b(a|an)\s+ordinal\s+score", re.I)
_WORDS = {"one": 1, "a": 1, "an": 1, "two": 2, "three": 3, "four": 4, "five": 5, "six": 6}


def _src26():
    with open(EXAMPLE_26, encoding="utf-8") as fh:
        return fh.read()


def _triage_type_counts():
    from laya.presets import triage_questions
    q = triage_questions()
    by_type = {}
    for spec in q.values():
        by_type[spec["type"]] = by_type.get(spec["type"], 0) + 1
    return len(q), by_type


def test_triage_preset_is_one_choice_three_noul_one_score():
    # The gate's positive anchor: this is the schema the banner must describe. If `triage_questions()`
    # ever moves, the banner test fails here first, and the drift is loud rather than silent.
    n_total, by_type = _triage_type_counts()
    assert n_total == 5, "triage_questions() no longer returns five fields: %r" % (by_type,)
    assert by_type == {"choice": 1, "noul": 3, "score": 1}, \
        "triage_questions()'s type mix moved: %r" % (by_type,)


def test_example_26_banner_counts_match_the_preset():
    n_total, by_type = _triage_type_counts()
    src = _src26()
    m_total = _STATED_TOTAL_26.search(src)
    m_yesno = _STATED_YESNO_26.search(src)
    m_choice = _STATED_CHOICE_26.search(src)
    m_score = _STATED_SCORE_26.search(src)
    assert m_total and m_yesno and m_choice and m_score, (
        "the banner no longer spells out its primitive counts; the gate needs them readable")
    assert _WORDS[m_total.group(1).lower()] == n_total, \
        "banner says %r questions, triage_questions() returns %d" % (m_total.group(1), n_total)
    assert _WORDS[m_yesno.group(1).lower()] == by_type["noul"], \
        "banner says %r yes/no probabilities, triage_questions() returns %d noul fields" % \
        (m_yesno.group(1), by_type["noul"])
    assert _WORDS[m_choice.group(1).lower()] == by_type["choice"]
    assert _WORDS[m_score.group(1).lower()] == by_type["score"]
    # Parts must add up to the whole the same paragraph asserts.
    parts = (_WORDS[m_choice.group(1).lower()] + _WORDS[m_yesno.group(1).lower()]
             + _WORDS[m_score.group(1).lower()])
    assert parts == _WORDS[m_total.group(1).lower()], (
        "banner arithmetic is wrong: %d + %d + %d = %d but it says %d questions" % (
            _WORDS[m_choice.group(1).lower()], _WORDS[m_yesno.group(1).lower()],
            _WORDS[m_score.group(1).lower()], parts, _WORDS[m_total.group(1).lower()]))


def test_example_26_drops_the_wrong_yes_no_count():
    # The ban has teeth: it fires on the pre-fix wording shipped on main.
    assert _BAN_TWO_YESNO.search(OLD_BANNER_26), "the ban must fire on the pre-fix wording"
    # ...and every count regex still reads the OLD banner as its pre-fix numbers, so the positive
    # rule was genuinely falsifiable.
    m = _STATED_YESNO_26.search(OLD_BANNER_26)
    assert m is not None and _WORDS[m.group(1).lower()] == 2
    # The current example satisfies the ban.
    assert not _BAN_TWO_YESNO.search(_src26()), "the wrong yes/no count is back in the shipped example"


for _fn in (test_triage_preset_is_one_choice_three_noul_one_score,
            test_example_26_banner_counts_match_the_preset,
            test_example_26_drops_the_wrong_yes_no_count):
    try:
        _fn()
    except AssertionError as e:
        FAIL.append("page-26/%s: %s" % (_fn.__name__, e))
    except Exception as e:                      # a crash is a failure, never a silent pass
        FAIL.append("page-26/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
    else:
        PASS.append("page-26/%s" % _fn.__name__)

# ------------------------------------------- tl_kernels' docstring must name the caller's real padding policy
# `laya/tl_kernels.py`'s module docstring said "M must be a multiple of 16 (the caller pads);
# out-of-bounds rows are predicated by TileLang". The caller pads two dimensions, not one:
# `laya/backends/base.py::bucket_rows` pads the batch dim to the next power of two (1, 2, 4, 8,
# 16, 32, ...) and `bucket_tokens` pads the sequence dim to a 16-token bucket up to
# `DYNAMIC_MAX_L` and a 64-token bucket beyond. M is the flattened `rows * tokens`, so the
# multiple-of-16 came from the token dim, never from the row dim. A reader who took the
# docstring as a row constraint would add `assert M % 16 == 0` at the call site and break
# every batch of 1, 2, 4, or 8 questions -- and the shipped `pad_batch` tests already pin those
# shapes: `tests/test_backends.py:86` "pad_batch/one row stays one row" is (1, 16), and
# `:89` "pad_batch/five rows pad to eight" is (8, 16).
TL_KERNELS = os.path.join(ROOT, "laya", "tl_kernels.py")
OLD_TL_DOCSTRING = (
    "GPU kernels take 16-bit activations (bf16 by default, fp16 with dtype=\"float16\"), accumulate in fp32.  Row count M is a runtime\n"
    "symbol so one compiled kernel serves every batch/sequence bucket; M must be a\n"
    "multiple of 16 (the caller pads); out-of-bounds rows are predicated by TileLang.\n"
    "Use compile_cpu for an explicit fp32 CPU specialization; the GPU defaults are unchanged."
)
_BAN_ROW_MULT_16 = re.compile(r"M must be a\s*\n?\s*multiple of 16 \(the caller pads\)", re.I)
_BAN_MULT_16_FROM_ROW = re.compile(r"multiple[- ]of[- ]16[^.\n]{0,80}from the row dim", re.I)
_NAMED_POWER_OF_TWO = re.compile(r"power of two", re.I)
_NAMED_16_TOKEN_BUCKET = re.compile(r"16[- ]token bucket", re.I)
_NAMED_NOT_ROW_DIM = re.compile(r"not the row dim", re.I)
_NAMED_BUCKET_ROWS = re.compile(r"\bbucket_rows\b")
_NAMED_BUCKET_TOKENS = re.compile(r"\bbucket_tokens\b")


def _tl_docstring():
    """Return `laya/tl_kernels.py`'s module docstring via `ast`, so the gate reads what a
    reader reads -- not whatever happens to appear in a comment."""
    with open(TL_KERNELS, encoding="utf-8") as fh:
        src = fh.read()
    return ast.get_docstring(ast.parse(src)) or ""


def test_tl_kernels_drops_the_wrong_row_claim():
    # The ban fires on the pre-fix docstring...
    assert _BAN_ROW_MULT_16.search(OLD_TL_DOCSTRING), "the ban must fire on the pre-fix wording"
    # ...and does not fire on the shipped one.
    doc = _tl_docstring()
    assert doc, "the tl_kernels module docstring was removed -- the gate has nothing to read"
    assert not _BAN_ROW_MULT_16.search(doc), (
        "`laya/tl_kernels.py` again says M must be a multiple of 16 because the caller pads; "
        "the caller pads rows to a power of two (1, 2, 4, 8, ...). A reader who trusts the "
        "claim asserts `M % 16 == 0` at the call site and breaks every small batch.")
    # The second ban catches the mis-attribution (claiming the 16-multiple comes from rows).
    assert not _BAN_MULT_16_FROM_ROW.search(doc), (
        "the docstring attributes the multiple-of-16 to the row dim; `bucket_rows` pads to a "
        "power of two, so the 16-multiple is a token-dim artefact.")


def test_tl_kernels_names_the_real_padding_policy():
    doc = _tl_docstring()
    assert _NAMED_POWER_OF_TWO.search(doc), (
        "the docstring must name the row pad as a power of two, matching `bucket_rows`")
    assert _NAMED_16_TOKEN_BUCKET.search(doc), (
        "the docstring must name the token pad as a 16-token bucket, matching `bucket_tokens`")
    assert _NAMED_BUCKET_ROWS.search(doc), (
        "the docstring must attribute the row pad to `bucket_rows`, the function that does it")
    assert _NAMED_BUCKET_TOKENS.search(doc), (
        "the docstring must attribute the token pad to `bucket_tokens`, the function that does it")
    assert _NAMED_NOT_ROW_DIM.search(doc), (
        "the docstring must name the row dim as the wrong source of the 16-multiple, so a "
        "reader cannot re-attribute it back")


def test_bucket_rows_stays_a_power_of_two_below_16():
    # The witness: rows below 16 really do stay at 1, 2, 4, 8, so the pre-fix claim
    # ("M must be a multiple of 16 (the caller pads)") was false for them. If `bucket_rows`
    # ever starts rounding to 16, the docstring fix needs review and this test says so.
    from laya.backends.base import bucket_rows
    for n, want in ((1, 1), (2, 2), (3, 4), (4, 4), (5, 8), (7, 8), (8, 8), (9, 16),
                    (15, 16), (17, 32), (24, 32), (33, 64)):
        got = bucket_rows(n)
        assert got == want, (
            "bucket_rows(%d) returned %d, expected %d -- the docstring's row contract needs "
            "review" % (n, got, want))
        assert got & (got - 1) == 0, (
            "bucket_rows(%d) = %d is not a power of two; the docstring's `power of two` claim "
            "is stale" % (n, got))
    # Rows below 16 must stay below 16 -- that is the case the pre-fix docstring got wrong.
    for n in (1, 2, 4, 8):
        assert bucket_rows(n) == n, (
            "bucket_rows(%d) = %d; a caller that trusts the old multiple-of-16 claim would "
            "expect 16 here" % (n, bucket_rows(n)))


# ------------------------------- and the caller it names has to be on the kernels' own route
# The attribution was the second half of the same defect. The docstring called
# `laya.backends.base.pad_batch` "the caller" of these kernels, but `pad_batch` serves the
# `backend="compile"` route: it has exactly one production call site, `laya/backends/compile.py:111`,
# and `laya/fast.py:10` is the only module in the package that imports `tl_kernels`. Those two sets
# being disjoint is what makes the sentence false, and nothing in the tree checked either one. The
# padding on the TileLang route happens in `FastLaya._forward_unlocked`, which takes its row bucket
# from `bucket_rows` (through `_bucket_n`) but *inlines* the token bucketing against its own
# `FastLaya.DYNAMIC_MAX_L`/`LONG_BUCKET` class attributes rather than calling `bucket_tokens`. So a
# maintainer who retuned `pad_batch` or `bucket_tokens` for TileLang padding changed nothing on the
# TileLang route, and no test said so. All three checks below read the call graph out of the AST --
# no `tilelang` import, so they run on a CPU-only box where the fast path cannot execute.

FAST_REL = "laya/fast.py"
COMPILE_REL = "laya/backends/compile.py"
_BAN_PAD_BATCH_CALLER = re.compile(r"caller(?:,| in) [`\w.]*pad_batch", re.I)


def _laya_trees():
    """`[(slash-relative path, parsed AST)]` for every module under `laya/`."""
    out = []
    for base, _dirs, files in os.walk(os.path.join(ROOT, "laya")):
        for name in sorted(files):
            if not name.endswith(".py") or "__pycache__" in base:
                continue
            path = os.path.join(base, name)
            with open(path, encoding="utf-8") as fh:
                out.append((path[len(ROOT) + 1:].replace(os.sep, "/"), ast.parse(fh.read(), filename=path)))
    return out


def _modules_calling(trees, fname):
    """Paths of the `laya/` modules containing a call node for `fname`."""
    hits = set()
    for rel, tree in trees:
        for node in ast.walk(tree):
            func = node.func if isinstance(node, ast.Call) else None
            if func is not None and ((isinstance(func, ast.Name) and func.id == fname)
                                     or (isinstance(func, ast.Attribute) and func.attr == fname)):
                hits.add(rel)
    return hits


def _modules_importing(trees, mod):
    """Paths of the `laya/` modules that import `mod`, `from . import mod` included."""
    hits = set()
    for rel, tree in trees:
        for node in ast.walk(tree):
            if isinstance(node, ast.ImportFrom) and any(a.name == mod for a in node.names):
                hits.add(rel)
            elif isinstance(node, ast.Import):
                if any(a.name.split(".")[-1] == mod for a in node.names):
                    hits.add(rel)
    return hits


def _unparsed_function(rel, funcname):
    """One function's body, `ast.unparse`d, so the check reads the code rather than its comments."""
    with open(os.path.join(ROOT, rel), encoding="utf-8") as fh:
        tree = ast.parse(fh.read(), filename=rel)
    for node in ast.walk(tree):
        if isinstance(node, ast.FunctionDef) and node.name == funcname:
            return ast.unparse(node)
    raise AssertionError("%s has no function %s() -- the docstring cites a moved symbol" % (rel, funcname))


def _class_constant(rel, classname, const):
    """A class-level `CONST = <literal>`, read without importing the module (this one needs tilelang)."""
    with open(os.path.join(ROOT, rel), encoding="utf-8") as fh:
        tree = ast.parse(fh.read(), filename=rel)
    for node in ast.walk(tree):
        if isinstance(node, ast.ClassDef) and node.name == classname:
            for stmt in node.body:
                if isinstance(stmt, ast.Assign):
                    for target in stmt.targets:
                        if isinstance(target, ast.Name) and target.id == const:
                            if isinstance(stmt.value, ast.Constant):
                                return stmt.value.value
    raise AssertionError("no `%s.%s = <literal>` in %s -- the docstring cites a moved constant"
                         % (classname, const, rel))


def test_tl_kernels_names_a_caller_that_is_on_the_kernels_route():
    """`pad_batch` never reaches these kernels, so it cannot be the caller the docstring names."""
    trees = _laya_trees()
    kernel_importers = _modules_importing(trees, "tl_kernels")
    pad_batch_callers = _modules_calling(trees, "pad_batch")
    assert kernel_importers == {FAST_REL}, (
        "the `laya/` modules importing `tl_kernels` are now %s; the docstring describes one caller, "
        "so a second route needs its own sentence" % sorted(kernel_importers))
    assert pad_batch_callers == {COMPILE_REL}, (
        "`pad_batch` is now called from %s, not only from `%s`; if a route from there reaches "
        "`tl_kernels`, the docstring's caller may genuinely be `pad_batch` again -- re-read both "
        "before editing" % (sorted(pad_batch_callers), COMPILE_REL))
    assert not (kernel_importers & pad_batch_callers), (
        "`pad_batch` is on the `tl_kernels` route after all, so this ban is stale -- re-read the "
        "call graph rather than deleting the sentence it caught")
    doc = _tl_docstring()
    assert not _BAN_PAD_BATCH_CALLER.search(doc), (
        "`laya/tl_kernels.py` again calls `laya.backends.base.pad_batch` the caller of these "
        "kernels. Its one production call site is `laya/backends/compile.py:111`, and "
        "`laya/fast.py:10` is the only module that imports `tl_kernels`; the padding these kernels "
        "see happens in `FastLaya._forward_unlocked`.")
    assert "laya.fast" in doc and "_forward_unlocked" in doc, (
        "the docstring must name the caller the call graph actually gives these kernels")


def test_the_fast_path_still_inlines_the_token_bucket_the_docstring_says_it_repeats():
    """`_forward_unlocked` mirrors `bucket_tokens`'s arithmetic against its own constants."""
    body = _unparsed_function(FAST_REL, "_forward_unlocked")
    assert "self.DYNAMIC_MAX_L" in body and "self.LONG_BUCKET" in body, (
        "`_forward_unlocked` no longer buckets tokens against the `FastLaya` class constants, so "
        "the docstring's 'repeats against its own `FastLaya.DYNAMIC_MAX_L`/`LONG_BUCKET`' is stale")
    assert "bucket_tokens(" not in body, (
        "`_forward_unlocked` calls `bucket_tokens` directly now; then there is one policy rather "
        "than two copies, and both the docstring and this check need re-reading")
    assert "_bucket_n(" in body or "bucket_rows(" in body, (
        "the row pad no longer comes from `laya.backends.base.bucket_rows`, through `_bucket_n` or "
        "directly, so the docstring's attribution needs re-reading")


def test_the_two_bucket_policies_the_docstring_describes_are_still_the_same_numbers():
    """One sentence covers both routes only while the duplicated constants agree."""
    from laya.backends.base import DYNAMIC_MAX_L, LONG_BUCKET
    fast_max_l = _class_constant(FAST_REL, "FastLaya", "DYNAMIC_MAX_L")
    fast_long = _class_constant(FAST_REL, "FastLaya", "LONG_BUCKET")
    assert (fast_max_l, fast_long) == (DYNAMIC_MAX_L, LONG_BUCKET), (
        "`FastLaya` buckets at (%s, %s) while `laya.backends.base` buckets at (%s, %s). "
        "`laya/tl_kernels.py`'s docstring describes one policy for these kernels; once the two "
        "copies differ it has to name both, and its `16`/`%d` numbers need re-reading."
        % (fast_max_l, fast_long, DYNAMIC_MAX_L, LONG_BUCKET, LONG_BUCKET))
    doc = _tl_docstring()
    assert "16-token bucket" in doc, "the docstring no longer states the small token bucket"
    assert "%d-token bucket" % LONG_BUCKET in doc, (
        "the docstring no longer states the large token bucket the code uses (`%d`)" % LONG_BUCKET)


for _fn in (test_tl_kernels_drops_the_wrong_row_claim,
            test_tl_kernels_names_the_real_padding_policy,
            test_bucket_rows_stays_a_power_of_two_below_16,
            test_tl_kernels_names_a_caller_that_is_on_the_kernels_route,
            test_the_fast_path_still_inlines_the_token_bucket_the_docstring_says_it_repeats,
            test_the_two_bucket_policies_the_docstring_describes_are_still_the_same_numbers):
    try:
        _fn()
    except AssertionError as e:
        FAIL.append("tl-kernels/%s: %s" % (_fn.__name__, e))
    except Exception as e:                      # a crash is a failure, never a silent pass
        FAIL.append("tl-kernels/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
    else:
        PASS.append("tl-kernels/%s" % _fn.__name__)

# --------------------------------------- DecisionResult's docstring must split confidence by question type
# `laya/structured.py`'s `DecisionResult` closed with "confidence keeps the **normalized-entropy
# value** it has always had, because that is a different quantity on a scale that depends on the
# label count". `Agent._decode_answers` (`laya/agent.py:1408-1437`) uses two different formulas:
#   * `choice` and `score`  ->  `confidence_from_probs(p, k)` = `1 - H(p) / log(k)`
#   * `noul`                ->  `max(p_true, 1 - p_true)`  (its own comment: "identical here:
#                               over two options `max(p_true, 1 - p_true)` is `max(p)`")
# The `noul` branch is deliberately not entropy: with k=2, entropy 1 - H/log(2) hits its floor
# (0.0) exactly when the answer is a coin flip, which is inverted from what a boolean reporter
# wants. `structured._details` (`:303`) copies `answer["confidence"]` verbatim, so
# `DecisionResult.confidence` is per-type -- the docstring's blanket "normalized-entropy"
# claim is false for every boolean field a caller projects. Same defect class as examples 18/40
# (#972, #973) but in the public docstring rather than in an example banner.
STRUCTURED = os.path.join(ROOT, "laya", "structured.py")
OLD_DECISIONRESULT_DOC = """The detailed result of `decide(..., return_details=True)`.

    `confidence` keeps the normalized-entropy value it has always had, because that is a
    different quantity on a scale that depends on the label count. A field that reported no
    usable `answer_confidence` maps to `None`, which is not the same as a reported `0.0`.
"""
_BAN_ENTROPY_UNIFORM = re.compile(r"`confidence` keeps the normalized-entropy value", re.I)
_NAMED_MAX_PT = re.compile(r"max\(p_true,\s*1 - p_true\)")
_NAMED_ENTROPY_FORMULA = re.compile(r"1 - H\(p\) / log\(k\)")
_NAMED_CHOICE_SCORE = re.compile(r"`choice` and\s+`score`", re.I)
_NAMED_NOUL = re.compile(r"for\s+`noul`", re.I)
_NAMED_NOT_ENTROPY = re.compile(r"not an\s+entropy", re.I)


def _decisionresult_doc():
    """Return the `DecisionResult` class docstring via `ast`, so the gate reads what a caller
    reads at `help(laya.structured.DecisionResult)` time."""
    with open(STRUCTURED, encoding="utf-8") as fh:
        src = fh.read()
    for node in ast.walk(ast.parse(src)):
        if isinstance(node, ast.ClassDef) and node.name == "DecisionResult":
            return ast.get_docstring(node) or ""
    return ""


def test_decisionresult_drops_the_uniform_entropy_claim():
    assert _BAN_ENTROPY_UNIFORM.search(OLD_DECISIONRESULT_DOC), (
        "the ban must fire on the pre-fix wording")
    doc = _decisionresult_doc()
    assert doc, "the DecisionResult class docstring was removed -- the gate has nothing to read"
    assert not _BAN_ENTROPY_UNIFORM.search(doc), (
        "`laya/structured.py`'s `DecisionResult` again claims `confidence` is uniformly the "
        "normalized entropy. `noul` fields carry `max(p_true, 1 - p_true)`, a probability, "
        "which is not an entropy and does not scale with the label count.")


def test_decisionresult_names_both_confidence_formulas():
    doc = _decisionresult_doc()
    assert _NAMED_ENTROPY_FORMULA.search(doc), (
        "the docstring must name the entropy formula `1 - H(p) / log(k)`")
    assert _NAMED_MAX_PT.search(doc), (
        "the docstring must name `max(p_true, 1 - p_true)` for the noul branch")
    assert _NAMED_CHOICE_SCORE.search(doc), (
        "the docstring must attribute the entropy formula specifically to `choice` and `score`")
    assert _NAMED_NOUL.search(doc), (
        "the docstring must attribute the probability formula specifically to `noul`")
    assert _NAMED_NOT_ENTROPY.search(doc), (
        "the docstring must state that the `noul` value is not an entropy, so a caller cannot "
        "silently re-collapse the two again")


def test_noul_and_entropy_confidence_are_different_quantities():
    # The witness: on the same 2-class distribution `confidence_from_probs` and the shipped
    # `noul` formula disagree sharply, so a docstring that says "confidence is normalized
    # entropy" and cites label count is wrong by construction. If they ever converge (say,
    # if `noul` switched to entropy), the docstring fix would need review and this test
    # tells the reader.
    uniform = np.array([0.5, 0.5])
    entropy_val = confidence_from_probs(uniform, 2)
    noul_val = max(float(uniform[1]), 1.0 - float(uniform[1]))
    assert entropy_val == 0.0 and noul_val == 0.5, (
        "the two formulas must disagree at the uniform 2-class: entropy=%r, noul=%r; if "
        "they've converged, the docstring's per-type split is stale" % (entropy_val, noul_val))
    # And on a peaked 2-class, entropy-confidence stays near 1 but is not the reported probability.
    peak = np.array([0.01, 0.99])
    entropy_peak = confidence_from_probs(peak, 2)
    noul_peak = max(float(peak[1]), 1.0 - float(peak[1]))
    assert abs(noul_peak - 0.99) < 1e-6 and abs(entropy_peak - 0.9192) < 1e-3, (
        "at (0.01, 0.99): entropy=%r (expect ~0.919) noul=%r (expect 0.99) -- if the values "
        "moved, the docstring's formula names need re-checking" % (entropy_peak, noul_peak))


for _fn in (test_decisionresult_drops_the_uniform_entropy_claim,
            test_decisionresult_names_both_confidence_formulas,
            test_noul_and_entropy_confidence_are_different_quantities):
    try:
        _fn()
    except AssertionError as e:
        FAIL.append("structured-doc/%s: %s" % (_fn.__name__, e))
    except Exception as e:                      # a crash is a failure, never a silent pass
        FAIL.append("structured-doc/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
    else:
        PASS.append("structured-doc/%s" % _fn.__name__)

# --------------------------------------- `docs/questions-and-answers.md` must count what `laya` exports
# The Presets section opens with "Three ready-made question sets" and its example imports three
# names, but `laya/__init__.py` re-exports five `*_questions` helpers -- `triage_questions`,
# `email_questions`, `guard_questions`, `moderation_questions`, `router_questions` -- and every one
# is in `laya.__all__`. A caller who reads "three" and picks from the shipped list misses two of
# the five; the doc drifts silently as presets get added because nothing ties the number to the
# exports. This gate reads the actual `laya.__all__` inventory so any future preset -- e.g. the
# Swedish sets on #729 -- fails the doc until the doc is updated alongside the export.
DOC_QA = os.path.join(ROOT, "docs", "questions-and-answers.md")
OLD_PRESET_INTRO = "Three ready-made question sets, so the common cases do not need hand-written criteria:"
_NUMBER_WORDS = {1: "One", 2: "Two", 3: "Three", 4: "Four", 5: "Five",
                 6: "Six", 7: "Seven", 8: "Eight", 9: "Nine", 10: "Ten"}


def _preset_exports():
    """The names `laya` publicly exports that end in `_questions`.

    Read via `laya.__all__`, not `dir(laya)` -- a helper that exists but is not in the export
    tuple is an internal name and does not need a doc mention.
    """
    import laya
    return sorted(n for n in laya.__all__ if n.endswith("_questions"))


def _doc_preset_section():
    with open(DOC_QA, encoding="utf-8") as fh:
        src = fh.read()
    idx = src.find("## Presets")
    if idx == -1:
        return ""
    nxt = src.find("\n## ", idx + len("## Presets"))
    return src[idx:] if nxt == -1 else src[idx:nxt]


def _doc_stated_count_word(section):
    m = re.search(r"\b(One|Two|Three|Four|Five|Six|Seven|Eight|Nine|Ten)\s+ready-made\s+question", section, re.I)
    return m.group(1) if m else None


def test_presets_page_drops_the_three_set_claim():
    # The witness: the pre-fix wording is banned, so if the doc ever regresses to "Three"
    # while the exports are still five, the gate fires.
    assert "Three ready-made" in OLD_PRESET_INTRO, (
        "the ban's literal must itself carry the wrong count, otherwise the ban is vacuous")
    section = _doc_preset_section()
    assert section, "the Presets section was removed -- the gate has nothing to read"
    word = _doc_stated_count_word(section)
    actual = len(_preset_exports())
    expected_word = _NUMBER_WORDS.get(actual)
    assert word == expected_word, (
        "docs/questions-and-answers.md's Presets section says %r but `laya.__all__` exports "
        "%d `*_questions` helpers (%s). Update the doc's number word to %r (or, if it has "
        "grown past the mapped words, extend `_NUMBER_WORDS`)." %
        (word, actual, ", ".join(_preset_exports()), expected_word))


def test_presets_page_names_every_exported_preset():
    section = _doc_preset_section()
    exports = _preset_exports()
    assert exports, "`laya.__all__` has no `*_questions` names -- either the exports moved or " \
                    "this gate is checking the wrong thing"
    missing = [n for n in exports if ("`" + n + "`") not in section]
    assert not missing, (
        "docs/questions-and-answers.md's Presets section does not name these exported helpers "
        "in backticks: %s. A caller who reads only this page cannot discover them." % missing)


def test_presets_page_no_three_claim_witness():
    # Direct ban on the pre-fix wording, independent of the count-regex, so a future PR that
    # quietly reverts to "Three" fails with an explicit "this phrase is banned" message rather
    # than a numeric-mismatch one.
    section = _doc_preset_section()
    assert "Three ready-made question sets" not in section, (
        "`docs/questions-and-answers.md` again says \"Three ready-made question sets\". As of "
        "this PR `laya.__all__` exports five `*_questions` helpers; if the number has since "
        "changed, update the doc's number word AND this ban's literal together.")


for _fn in (test_presets_page_drops_the_three_set_claim,
            test_presets_page_names_every_exported_preset,
            test_presets_page_no_three_claim_witness):
    try:
        _fn()
    except AssertionError as e:
        FAIL.append("docs-presets/%s: %s" % (_fn.__name__, e))
    except Exception as e:                      # a crash is a failure, never a silent pass
        FAIL.append("docs-presets/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
    else:
        PASS.append("docs-presets/%s" % _fn.__name__)

# --- jev_confidence: Jev's `confidence` formula, reported beside Laya's own (#302) ----------
# The vectors are TypeSafe's own (system-one-adapter-python, tests/utils/test_confidence_metrics.py)
# plus the two answers in the jevcompat spec's example response.
from laya.confidence import jev_confidence  # noqa: E402


def _choice(*p):
    return {"type": "choice", "probabilities": {"o%d" % i: v for i, v in enumerate(p)}}


def _score(*p):
    return {"type": "score", "probabilities": {str(i): v for i, v in enumerate(p)}}


for _name, _answer, _want in (
    ("choice/uniform-2", _choice(0.5, 0.5), 0.0),
    ("choice/zero-total-is-uniform", _choice(0.2, 0.2), 0.0),
    ("choice/0.82-0.18", _choice(0.82, 0.18), 0.64),
    ("choice/single-option", _choice(1.0), 1.0),
    ("choice/jevcompat-example", _choice(0.88, 0.12), 0.76),
    ("score/uniform-5", _score(*[0.2] * 5), 0.0),
    ("score/unnormalized-uniform-5", _score(*[0.04] * 5), 0.0),
    ("score/peaked-5", _score(0.01, 0.02, 0.07, 0.3, 0.6), 0.55),
    ("score/single-level", _score(1.0), 1.0),
    ("score/jevcompat-example", _score(0.0, 0.95, 0.05), 0.925),
):
    _got = jev_confidence(_answer)
    check_true("jev_confidence/%s" % _name, _got is not None and abs(_got - _want) < 1e-9,
               "got %r, want %r" % (_got, _want))

# --- finite total overflow: a common scale must not change the confidence -----------------
# Each entry below is finite and non-negative. The public helper accepts unnormalised weights,
# but their sum can still overflow: dividing by infinity turns them all into zeros, reporting
# negative choice confidence or maximum score confidence for an uncertain distribution.
_JEV_FINITE_SCALE_CASES = (
    ("overflow-uniform", (1e308, 1e308), (1.0, 1.0)),
    ("overflow-skewed", (1e308, 8e307), (1.0, 0.8)),
    ("overflow-with-zero", (0.0, 1e308, 1e308), (0.0, 1.0, 1.0)),
    ("overflow-three", (1e308, 5e307, 5e307), (1.0, 0.5, 0.5)),
    ("overflow-integers", (10 ** 308, 10 ** 308), (1.0, 1.0)),
    ("overflow-numeric-strings", ("1e308", "1e308"), (1.0, 1.0)),
    ("small-normal", (1e-308, 8e-309), (1.0, 0.8)),
    ("smallest-subnormal", (5e-324, 5e-324), (1.0, 1.0)),
    ("zero-total", (0.0, 0.0), (1.0, 1.0)),
)
for _name, _weights, _reference in _JEV_FINITE_SCALE_CASES:
    for _build in (_choice, _score):
        _answer = _build(*_weights)
        _before = copy.deepcopy(_answer)
        _got = jev_confidence(_answer)
        _want = jev_confidence(_build(*_reference))
        check_true("jev_finite_scale/%s/%s" % (_answer["type"], _name),
                   _got is not None and abs(_got - _want) < 1e-9,
                   "got %r, want %r" % (_got, _want))
        check_true("jev_finite_range/%s/%s" % (_answer["type"], _name),
                   _got is not None and 0.0 <= _got <= 1.0, repr(_got))
        check("jev_finite_input/%s/%s" % (_answer["type"], _name), _answer, _before)
# --- end finite total overflow -------------------------------------------------------------

# Score levels are read in numeric order, not string order: "10" sorts before "2" as text.
_levels = {str(i): 0.0 for i in range(11)}
_levels["10"] = 1.0
check("jev_confidence/score-levels-in-numeric-order",
      jev_confidence({"type": "score", "probabilities": _levels}), 1.0)

# The same distribution gives the two numbers #302 measured apart: entropy is not Jev's formula.
_p = np.array([0.7, 0.1, 0.1, 0.1])
check_true("jev_confidence/differs-from-entropy-confidence",
           abs(jev_confidence(_choice(*_p)) - 0.6) < 1e-9 and confidence_from_probs(_p, 4) < 0.35,
           "jev %r, entropy %r" % (jev_confidence(_choice(*_p)), confidence_from_probs(_p, 4)))

# No number rather than a made-up one: a noul has no Jev confidence, and malformed input is refused.
for _name, _answer in (
    ("noul", {"type": "noul", "noul": 0.9}),
    ("unknown-type", {"type": "weird", "probabilities": {"a": 1.0}}),
    ("missing-probabilities", {"type": "choice"}),
    ("empty-probabilities", {"type": "choice", "probabilities": {}}),
    ("non-numeric-probability", _choice(0.5, "high")),
    ("bool-probability", _choice(True, False)),
    ("nan-probability", _choice(float("nan"), 0.5)),
    ("infinite-probability", _choice(float("inf"), 0.5)),
    ("non-integer-score-level", {"type": "score", "probabilities": {"low": 0.5, "high": 0.5}}),
    # A negative probability is malformed, and at `432312b3` none of these five were refused: they
    # returned 3.0, 2.0, 0.8, 121.0 (120.99999999999989) and 1.75. The third is the one that cannot
    # be caught by inspection -- an all-negative document divides by its own negative total and
    # comes back with the number its positive counterpart comes back with.
    ("negative-probability", _choice(2.0, -1.0)),
    ("mixed-sign-argmax-flips", _choice(0.5, -1.5)),
    ("all-negative-probabilities", _choice(-0.9, -0.1)),
    ("negative-total-near-zero", _choice(-0.61, 0.6)),
    ("negative-score-level", _score(1.5, -0.5, 0.0)),
    # `float()` and `int()` do not only refuse a value that is not a number, they also refuse one
    # that is too wide to become one: JSON's grammar has no double limit, so a 400-digit integer
    # probability arrives as a Python `int` and `float()` raises `OverflowError` out of a function
    # whose docstring promises no number rather than a made-up one.
    ("overflowing-integer-probability", _choice(10 ** 400, 1)),
    ("json-integer-too-wide-for-a-double",
     _json.loads('{"type":"choice","probabilities":{"a":' + "9" * 400 + ',"b":0.1}}')),
    ("overflowing-score-level", {"type": "score", "probabilities": {float("inf"): 0.5, 2: 0.5}}),
):
    check("jev_confidence/none-for-%s" % _name, jev_confidence(_answer), None)


# The table above refuses a document whose probabilities are *all* boolean, which is the one shape a
# `any(isinstance(v, bool) ...)` guard and an `all(...)` one agree on. The positions below are where
# the two disagree, and they are the shapes a client actually sends: `{"spam": true, "ham": 0.4}`
# mixes a JSON boolean with real numbers. Under `all` that document is measured -- `float(True)` is
# 1.0, so the answer comes back as the most confident option the caller never scored -- and
# `/v1/systemone` publishes the number as `x_jev_confidence` for a client to threshold against.
_swept, _measured = 0, []
for _k in (2, 3, 4, 5, 8, 12):
    for _i in range(_k):
        for _flag in (True, False):
            _vals = [1.0 / _k] * _k
            _vals[_i] = _flag
            for _answer in (_choice(*_vals), _score(*_vals)):
                _got = jev_confidence(_answer)
                _swept += 1
                if _got is not None:
                    _measured.append((_k, _i, _flag, _answer["type"], _got))
check_true("jev_confidence/a bool in any position of any option count is refused",
           not _measured, "%d of %d measured: %r" % (len(_measured), _swept, _measured[:3]))
# Every one of those documents differs from its all-float twin by exactly one entry, so the count
# is the witness that the sweep ran over the positions it claims.
check("jev_confidence/position sweep size", _swept, sum(2 * 2 * _k for _k in (2, 3, 4, 5, 8, 12)))


def _jev_measured(_answer):
    """`jev_confidence`'s number, or the exception it raised instead, so a slip reports, not aborts."""
    try:
        return jev_confidence(_answer)
    except BaseException as _exc:  # noqa: BLE001 -- the bare TypeError a misplaced check raises
        return _exc


# A probability written as the string "0.9" reads as a number through the function's own float()
# coercion, so it is measured rather than refused -- and the finite and range checks run on that
# coerced value, not on the JSON type it arrived as.
_got = _jev_measured(_choice("0.9", "0.1"))
check_true("jev_confidence/a numeric string probability is measured as a number",
           isinstance(_got, float) and abs(_got - 0.8) < 1e-9, repr(_got))

# The range the name promises: nothing this returns is outside [0, 1], and the negatives are
# exactly what is refused. `x_jev_confidence` is served for a client to compare a threshold
# against, so an out-of-range number is a gate that passes every answer it touches.
_swept, _out_of_range, _refused = 0, [], []
for _i in range(-200, 301):
    for _answer in (_choice(_i / 100.0, 0.6), _score(_i / 100.0, 0.4, 0.2)):
        _got = jev_confidence(_answer)
        _swept += 1
        if _got is None:
            _refused.append(_i / 100.0)
        elif not (0.0 <= _got <= 1.0):
            _out_of_range.append((_i / 100.0, _got))
check_true("jev_confidence/sweep kept every number inside [0, 1]", not _out_of_range,
           "%d of %d outside, worst %r" % (len(_out_of_range), _swept, _out_of_range[:3]))
check_true("jev_confidence/sweep refused exactly the negative probability",
           sorted(set(_refused)) == [value / 100.0 for value in range(-200, 0)],
           "%d refused: %r" % (len(_refused), _refused[:2] + _refused[-2:]))
# 501 values x both answer types, and the 400 refusals are the 200 negatives twice over
check_true("jev_confidence/sweep measured everything that is not negative",
           (_swept, len(_refused), _swept - len(_refused)) == (1002, 400, 602),
           "swept %d, refused %d, measured %d" % (_swept, len(_refused), _swept - len(_refused)))

# --- the public name and the served field must be one computation (#302) --------------------
# `/v1/systemone` publishes `x_jev_confidence`, but a caller running the checkpoint in-process
# gets an answer dict and no HTTP layer, and the other two confidence quantities
# (`answer_confidence`, `confidence_from_probs`) are already public. So the name ships from
# `laya`, and these checks pin that the served number is this function's number.
import laya  # noqa: E402

check_true("jev_public/exported", "jev_confidence" in laya.__all__)
check_true("jev_public/lazy attr resolves to the core function",
           getattr(laya, "jev_confidence", None) is jev_confidence)

from laya.serve import _add_jev_confidence  # noqa: E402

_answers = {
    "q": _choice(0.88, 0.12),
    "r": _choice(2, 1),          # 1/3, so the served rounding is visible
    "s": _score(0.0, 0.95, 0.05),
    "n": {"type": "noul", "noul": 0.9},
}
_served = _add_jev_confidence({"answers": copy.deepcopy(_answers)})["answers"]
for _qid in ("q", "r", "s"):
    check("jev_public/served %s is the public function rounded" % _qid,
          _served[_qid]["x_jev_confidence"], round(jev_confidence(_answers[_qid]), 4))
check_true("jev_public/served noul carries no field",
           "x_jev_confidence" not in _served["n"], _served["n"].keys())

# --- probability precision: both decoders round independently, not to a unit sum (#555) ------
from laya.agent import Agent  # noqa: E402
from laya.onnx_agent import ONNXAgent  # noqa: E402

for _cls in (Agent, ONNXAgent):
    _agent = _cls.__new__(_cls)  # no checkpoint, model or ONNX session
    _agent.temperature = [1.0, 1.0, 1.0]
    _agent.temperature_by_options = {}
    for _k, _value, _total in ((3, 0.3333, 0.9999), (6, 0.1667, 1.0002)):
        for _qtype in ("choice", "score"):
            _labels = ["o%d" % i for i in range(_k)]
            _crit = dict.fromkeys(_labels) if _qtype == "choice" else _labels
            _answer = _agent._decode_answers(
                np.zeros((1, _k)), np.array([[1.0, 0.0]]), [{"markers": list(range(_k))}],
                ["q"], {"q": {"t": _qtype, "crit": _crit}}, 0)["q"]
            _probs = list(_answer["probabilities"].values())
            _name = "rounding/%s/%s/k=%d" % (_cls.__name__, _qtype, _k)
            check(_name + "/four decimal places", _probs, [_value] * _k)
            check_true(_name + "/non-unit sum", close(sum(_probs), _total), repr(_probs))

for _page in ("questions-and-answers.md", "http-api.md"):
    _text = _prose(_read_page(os.path.join(DOC_ROOT, "docs", _page)))
    check_true("rounding/docs/%s explains precision" % _page,
               bool(_re.search(r"`probabilities`.*?rounded independently to four decimal places", _text)))
    check_true("rounding/docs/%s qualifies the unit sum" % _page,
               "need not sum exactly to 1" in _text)

_tournament = _prose(_read_page(os.path.join(DOC_ROOT, "docs", "http-api.md")).split(
    "### Large choices:", 1)[1].split("## Limits", 1)[0])
check_true("rounding/docs/tournament does not promise an exact unit sum",
           "need not sum exactly to 1" in _tournament)

print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
    print("  FAIL", f)
if not FAIL:
    print("all confidence tests passed")
sys.exit(1 if FAIL else 0)