Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #979 from aashish254/fix-tl-kernels-row-padding

Nandakishor13a01a42a076
diff --cc tests/test_confidence.py
index bac8c73e,255632ec..aca93f5a
--- a/tests/test_confidence.py
+++ b/tests/test_confidence.py
@@@ -1120,840 -603,105 +1120,939 @@@ for _fn in (test_preset_is_four_flags_a
else:
PASS.append("page-28/%s" % _fn.__name__)
+
+# --------------------------------------------- example 40 must not call a raw softmax calibrated
+# `examples/40_caching_and_monitoring.py` gates 15 tickets on `answer_confidence`. On main it calls
+# that field "the calibrated probability of the answer Laya reports" in both its banner and its
+# closing paragraph, and its `bucket()` docstring reads "application policy over a calibrated
+# number". `answer_confidence` is max(p) -- the quantity temperature scaling fits and ECE measures
+# -- but the README's own Calibration section says that reading holds *only* after temperatures are
+# fitted and validated, the shipped checkpoints are over-confident, and the loader emits at load
+# time `RuntimeWarning: ... choice:11+=0.10058... -> 0.5. Treat confidence from the affected
+# entries as uncalibrated.` The page now reads the temperature each shape was actually scaled by off
+# the loaded agent and states the calibration as conditional.
+#
+# No weights are loaded here: the four helpers are pulled from the example's AST and exec'd against
+# a fabricated agent, and `scale_for`'s lookup is checked against core's own `temp_bucket`, so the
+# gate drives the code the page runs instead of re-reading its prose.
+from laya.common import QTYPES as _QTYPES40, temp_bucket as _temp_bucket # noqa: E402
+
+EXAMPLE_40 = os.path.join(ROOT, "examples", "40_caching_and_monitoring.py")
+HELPERS_40 = ("option_count", "scale_for", "clamped_buckets", "entropy_confidence")
+# Names a helper may read from the page's world: the two core symbols it must defer to, and `math`
+# for the entropy. Any other non-builtin free name means this gate cannot drive that helper.
+CORE_GLOBALS_40 = {"temp_bucket", "QTYPES", "math"}
+
+# main's page, as literal source lines. Every ban is witnessed against this text; every positive
+# rule below is witnessed by showing this text does NOT satisfy it.
+OLD_PAGE_40 = '''
+ the triage preset and gates on `answer_confidence`, the calibrated probability of the
+ """Our thresholds, not the model's: application policy over a calibrated number."""
+ The monitor gates on `answer_confidence`: the calibrated probability of the answer Laya
+ distribution has H/log(k) = %.2f, so `confidence` is %.2f while
+ """ % (entropy, 1 - entropy, vague["intent"]["answer_confidence"]))
+'''
+
+UNCONDITIONAL_CALIBRATION = re.compile(r"the calibrated probability", re.I)
+POLICY_OVER_CALIBRATED = re.compile(r"over a calibrated number", re.I)
+SUBSTITUTED_CONFIDENCE = re.compile(r"1\s*-\s*entropy")
+DERIVES_SCALING = re.compile(r"temp_bucket\(")
+READS_REPORTED_CONFIDENCE = re.compile(r"\[[\"']confidence[\"']\]")
+HARDCODED_BUCKET = re.compile(r"[\"'](choice|score|noul):[0-9]")
+
+
+def _src40():
+ with open(EXAMPLE_40, encoding="utf-8") as fh:
+ return fh.read()
+
+
+def _helpers40():
+ """Exec only the example's top-level helpers -- its `load()` call needs weights."""
+ tree = ast.parse(_src40())
+ nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in HELPERS_40]
+ ns = {"temp_bucket": _temp_bucket, "QTYPES": _QTYPES40, "math": math}
+ exec(compile(ast.Module(body=nodes, type_ignores=[]), EXAMPLE_40, "exec"), ns) # noqa: S102
+ return ns
+
+
+def _called40():
+ tree = ast.parse(_src40())
+ top = [n for n in tree.body if not isinstance(
+ n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Import, ast.ImportFrom))]
+ return {node.id for stmt in top for node in ast.walk(stmt)
+ if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)}
+
+
+def _fn40(name):
+ for n in ast.parse(_src40()).body:
+ if isinstance(n, ast.FunctionDef) and n.name == name:
+ return n
+ raise KeyError(name)
+
+
+class _FakeAgent(object):
+ """Only the three temperature attributes `scale_for` and `clamped_buckets` read."""
+
+ def __init__(self, by_options, per_type, raw=None):
+ self.temperature_by_options = by_options
+ self.temperature = per_type
+ self.temperature_by_options_raw = dict(by_options) if raw is None else raw
+
+
+def test_page40_helpers_are_live_defer_to_core():
+ defined = sorted(n.name for n in ast.parse(_src40()).body
+ if isinstance(n, ast.FunctionDef) and n.name in HELPERS_40)
+ assert defined == sorted(HELPERS_40), "example 40 lost a helper: %s" % defined
+ ns = _helpers40()
+ unused = set(HELPERS_40) - _called40()
+ assert not unused, "defined but never called at module level: %s" % sorted(unused)
+ for name in HELPERS_40:
+ node = _fn40(name)
+ outside = sorted(n for n in _free_names(node)
+ if not hasattr(builtins, n) and n not in CORE_GLOBALS_40)
+ assert not outside, "%s reads %s from the page, so this gate cannot drive it" % (name, outside)
+ prints = [c for c in ast.walk(node)
+ if isinstance(c, ast.Call) and isinstance(c.func, ast.Name) and c.func.id == "print"]
+ assert not prints, "%s prints instead of returning" % name
+ # `scale_for` must resolve the bucket through core, not carry a typed-in table.
+ assert not HARDCODED_BUCKET.search(ast.dump(_fn40("scale_for"))), (
+ "scale_for hardcodes a bucket name instead of calling temp_bucket")
+
+
+def test_scale_for_replays_core_lookup():
+ """For every shape, scale_for returns exactly `temperature_by_options.get(bucket, per_type)`."""
+ ns = _helpers40()
+ per_type = [7.0, 8.0, 9.0] # choice, score, noul fallbacks
+ by_options = {"choice:6-10": 1.0, "noul:2": 1.9834, "score:3-5": 1.2514}
+ fake = _FakeAgent(by_options, per_type)
+ for qtype, k in (("choice", 6), ("noul", 2), ("score", 4), ("noul", 2)):
+ name, applied, in_map = ns["scale_for"](fake, qtype, k)
+ bucket = _temp_bucket(_QTYPES40[qtype], k)
+ assert name == bucket, (qtype, k, name, bucket)
+ assert in_map == (bucket in by_options), (bucket, in_map)
+ assert abs(applied - by_options.get(bucket, per_type[_QTYPES40[qtype]])) < 1e-9, (
+ "%s: %r not core's get(...)" % (bucket, applied))
+ # the fallback arm: an unmapped shape must fall to the per-type temperature, flagged not-in-map.
+ name, applied, in_map = ns["scale_for"](_FakeAgent({}, [0.5, 0.6, 0.7]), "choice", 12)
+ assert (name, applied, in_map) == ("choice:11+", 0.5, False), (name, applied, in_map)
+ # a mapped temperature of exactly 1.0 must be reported as 1.0, not nudged.
+ assert ns["scale_for"](_FakeAgent({"choice:6-10": 1.0}, [1.0, 1.0, 1.0]), "choice", 6) == (
+ "choice:6-10", 1.0, True)
+
+
+def test_option_count_matches_the_shipped_preset():
+ ns = _helpers40()
+ counts = {qid: ns["option_count"](q) for qid, q in laya.triage_questions().items()}
+ assert counts == {"intent": 6, "is_urgent": 2, "frustration": 4,
+ "refund_requested": 2, "churn_risk": 2}, counts
+
+
+def test_clamped_buckets_names_only_a_shipped_value_the_runtime_refused():
+ ns = _helpers40()
+ # `choice:11+` ships below TEMP_MIN and is clamped to 0.5; everything else ships unchanged.
+ applied = {"choice:11+": 0.5, "noul:2": 1.9834}
+ raw = {"choice:11+": 0.10058280825614929, "noul:2": 1.9834}
+ out = ns["clamped_buckets"](_FakeAgent(applied, [0.5, 0.5, 0.5], raw))
+ assert out == ["choice:11+=0.1006 -> 0.5000"], out
+ # and an honest checkpoint reports nothing.
+ same = {"noul:2": 1.9834}
+ assert ns["clamped_buckets"](_FakeAgent(same, [0.5, 0.5, 0.5], dict(same))) == []
+
+
+def test_entropy_confidence_recomputes_core_exactly():
+ ns = _helpers40()
+ for dist in ([0.5, 0.3, 0.2], [0.9, 0.1], [0.25, 0.25, 0.25, 0.25], [1.0, 0.0, 0.0]):
+ p = np.array(dist, dtype=float)
+ got = ns["entropy_confidence"](dist)
+ want = confidence_from_probs(p, len(dist))
+ assert close(got, want, 1e-9), (dist, got, want)
+ assert ns["entropy_confidence"]([1.0]) == 1.0, "k<2 is 1.0 by definition"
+
+
+def test_page40_drops_the_unconditional_calibration_claim():
+ """Each ban fires on main's page and not on this one -- a ban with no witness is a guess."""
+ src = _src40()
+ for name, rule in (("'the calibrated probability'", UNCONDITIONAL_CALIBRATION),
+ ("'over a calibrated number'", POLICY_OVER_CALIBRATED),
+ ("prints `1 - entropy` as the confidence", SUBSTITUTED_CONFIDENCE)):
+ assert rule.search(OLD_PAGE_40), "%s does not fire on the wording it bans" % name
+ assert not rule.search(src), "%s is still in example 40" % name
+
+
+def test_page40_reads_the_scaling_and_the_reported_field():
+ """The positive half: the page must derive the temperature and print the reported field."""
+ src = _src40()
+ assert DERIVES_SCALING.search(src), "the bucket must be resolved through core's temp_bucket"
+ assert READS_REPORTED_CONFIDENCE.search(src), "the page must read the reported `confidence`"
+ # and main's page satisfies none of that, so these rules could not have passed before.
+ assert not DERIVES_SCALING.search(OLD_PAGE_40)
+ assert not READS_REPORTED_CONFIDENCE.search(OLD_PAGE_40)
+
+
+for _fn in (test_page40_helpers_are_live_defer_to_core, test_scale_for_replays_core_lookup,
+ test_option_count_matches_the_shipped_preset,
+ test_clamped_buckets_names_only_a_shipped_value_the_runtime_refused,
+ test_entropy_confidence_recomputes_core_exactly,
+ test_page40_drops_the_unconditional_calibration_claim,
+ test_page40_reads_the_scaling_and_the_reported_field):
+ try:
+ _fn()
+ except AssertionError as e:
+ FAIL.append("page-40/%s: %s" % (_fn.__name__, e))
+ except Exception as e: # a crash is a failure, never a silent pass
+ FAIL.append("page-40/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
+ else:
+ PASS.append("page-40/%s" % _fn.__name__)
+# Gate: the two predict_long call sites' comments must describe the None-min_confidence
+# contract truthfully. The pre-fix wording claimed every answer "reports that it ran ungated"
+# / "say so on every answer"; confidence.py:196-197 documents and :211-212 implements the
+# opposite -- the gate writes nothing at all, so the payload carries no `abstention` field.
+import ast as _gate_ast # noqa: E402
+
+_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+def _gate_gate_comment(rel_path):
+ """Return the run of comment lines that immediately precede the
+ apply_confidence_gate([result], None) call inside predict_long."""
+ with open(os.path.join(_REPO, rel_path)) as f:
+ lines = f.readlines()
+ tree = _gate_ast.parse("".join(lines))
+ for node in _gate_ast.walk(tree):
+ if isinstance(node, _gate_ast.FunctionDef) and node.name == "predict_long":
+ call_line = None
+ for sub in _gate_ast.walk(node):
+ if (isinstance(sub, _gate_ast.Call)
+ and getattr(sub.func, "id", getattr(sub.func, "attr", None)) == "apply_confidence_gate"
+ and len(sub.args) == 2
+ and isinstance(sub.args[1], _gate_ast.Constant)
+ and sub.args[1].value is None):
+ call_line = sub.lineno - 1
+ break
+ if call_line is None:
+ continue
+ out = []
+ i = call_line - 1
+ while i >= 0 and lines[i].lstrip().startswith("#"):
+ out.insert(0, lines[i].strip().lstrip("#").strip())
+ i -= 1
+ return " ".join(out)
+ return ""
+
+
+_agent_comment = _gate_gate_comment(os.path.join("laya", "agent.py"))
+_onnx_comment = _gate_gate_comment(os.path.join("laya", "onnx_agent.py"))
+
+for _label, _comment in (("agent.py", _agent_comment), ("onnx_agent.py", _onnx_comment)):
+ check_true("gate/comment/%s exists" % _label, len(_comment) > 0, "no comment found")
+ check_true("gate/comment/%s drops the ungated-reports claim" % _label,
+ "reports that it ran ungated" not in _comment
+ and "say so on every answer" not in _comment,
+ "pre-fix wording still on the call site: %r" % _comment)
+ check_true("gate/comment/%s names writes nothing" % _label,
+ "writes nothing" in _comment,
+ "gate comment must name the None-contract: %r" % _comment)
+ check_true("gate/comment/%s names the min_confidence argument" % _label,
+ "min_confidence" in _comment,
+ "gate comment must refer to the argument it is about: %r" % _comment)
+
+# Live witness: apply_confidence_gate([payload], None) leaves the payload byte-identical,
+# so the corrected comment is not itself a claim the code contradicts.
+_witness = {"model": "w", "answers": {"q": {"choice": "a", "confidence": 0.42}}, "usage": {}}
+_witness_before = copy.deepcopy(_witness)
+apply_confidence_gate([_witness], None)
+check("gate/live None writes nothing", _witness, _witness_before)
+check("gate/live None adds no abstention key", "abstention" in _witness["answers"]["q"], False)
+
+
+# --------------------------------------------- example 18 must not call the threshold a measurement
+# `examples/18_confidence_gating.py` is the intro-level gating pattern: it picks a 0.85 cutoff on
+# `answer_confidence` and acts on it. On main it calls the field "the calibrated probability Laya
+# puts on the answer it reports" and its module docstring says it "branches on Laya's calibrated
+# confidence". The README's Calibration section (line 1586) says the opposite about the shipped
+# state: mean ECE on `laya` is 0.466 as shipped, dropping to 0.081 only after refitting one
+# temperature per (question type, option-count) bucket on held-out data, and both shipped
+# checkpoints are over-confident. `answer_confidence` is the field temperature scaling fits and
+# the abstention gate reads -- but "calibrated" is a property a fit earns, not a property of the
+# field, and the loader emits `RuntimeWarning: ... Treat confidence from the affected entries as
+# uncalibrated.` on this very checkpoint. The page now reads back the temperature it actually
+# applies to each of its two questions and states the calibration claim as conditional.
+from laya.common import QTYPES as _QTYPES18, temp_bucket as _temp_bucket18 # noqa: E402
+
+EXAMPLE_18 = os.path.join(ROOT, "examples", "18_confidence_gating.py")
+HELPERS_18 = ("option_count", "scale_for")
+CORE_GLOBALS_18 = {"temp_bucket", "QTYPES", "math"}
+
+OLD_PAGE_18 = '''
+Answers a batch of support emails and branches on Laya\'s calibrated confidence: act
+ The production pattern from the README. Gate on `answer_confidence`: the calibrated
+ probability Laya puts on the answer it reports, defined the same way on every question
+'''
+
+UNQUALIFIED_CALIBRATION_18 = re.compile(
+ r"the\s+calibrated\s+probability|Laya['’]s\s+calibrated\s+confidence", re.I)
+DERIVES_SCALING_18 = re.compile(r"temp_bucket\(")
+NAMES_CONDITION_18 = re.compile(r"over-confident", re.I)
+HARDCODED_BUCKET_18 = re.compile(r"[\"'](choice|score|noul):[0-9]")
+
+
+def _src18():
+ with open(EXAMPLE_18, encoding="utf-8") as fh:
+ return fh.read()
+
+
+def _helpers18():
+ tree = ast.parse(_src18())
+ nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in HELPERS_18]
+ ns = {"temp_bucket": _temp_bucket18, "QTYPES": _QTYPES18, "math": math}
+ exec(compile(ast.Module(body=nodes, type_ignores=[]), EXAMPLE_18, "exec"), ns) # noqa: S102
+ return ns
+
+
+class _FakeAgent18(object):
+ """Only the two temperature attributes `scale_for` reads."""
+
+ def __init__(self, by_options, per_type):
+ self.temperature_by_options = by_options
+ self.temperature = per_type
+
+
+def _top_literal18(name):
+ for n in ast.parse(_src18()).body:
+ if isinstance(n, ast.Assign) and any(
+ isinstance(t, ast.Name) and t.id == name for t in n.targets):
+ return ast.literal_eval(n.value)
+ raise KeyError(name)
+
+
+def _fn18(name):
+ for n in ast.parse(_src18()).body:
+ if isinstance(n, ast.FunctionDef) and n.name == name:
+ return n
+ raise KeyError(name)
+
+
+def test_page18_helpers_are_live_and_defer_to_core():
+ defined = sorted(n.name for n in ast.parse(_src18()).body
+ if isinstance(n, ast.FunctionDef) and n.name in HELPERS_18)
+ assert defined == sorted(HELPERS_18), "example 18 lost a helper: %s" % defined
+ tree = ast.parse(_src18())
+ top_loads = {node.id for stmt in tree.body if not isinstance(
+ stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Import, ast.ImportFrom))
+ for node in ast.walk(stmt)
+ if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)}
+ unused = set(HELPERS_18) - top_loads
+ assert not unused, "defined but never called at module level: %s" % sorted(unused)
+ for name in HELPERS_18:
+ node = _fn18(name)
+ outside = sorted(n for n in _free_names(node)
+ if not hasattr(builtins, n) and n not in CORE_GLOBALS_18)
+ assert not outside, "%s reads %s from the page" % (name, outside)
+ prints = [c for c in ast.walk(node)
+ if isinstance(c, ast.Call) and isinstance(c.func, ast.Name) and c.func.id == "print"]
+ assert not prints, "%s prints instead of returning" % name
+ assert not HARDCODED_BUCKET_18.search(ast.dump(_fn18("scale_for"))), (
+ "scale_for hardcodes a bucket name instead of calling temp_bucket")
+
+
+def test_page18_questions_reach_core_bucket_names():
+ """Replay the page's own QUESTIONS through core's temp_bucket and through the example's helper."""
+ ns = _helpers18()
+ questions = _top_literal18("QUESTIONS")
+ per_type = [7.0, 8.0, 9.0]
+ by_options = {"choice:3-5": 1.7601518630981445, "noul:2": 1.983399510383606}
+ fake = _FakeAgent18(by_options, per_type)
+ shapes = {}
+ for qid, q in questions.items():
+ k = ns["option_count"](q)
+ name, applied, in_map = ns["scale_for"](fake, q["type"], k)
+ assert name == _temp_bucket18(_QTYPES18[q["type"]], k), (qid, name)
+ assert abs(applied - by_options[name]) < 1e-9, (qid, applied)
+ assert in_map is True, "%s: %s missing from map" % (qid, name)
+ shapes[qid] = (q["type"], k, name, applied)
+ assert shapes == {
+ "department": ("choice", 4, "choice:3-5", 1.7601518630981445),
+ "refund_requested": ("noul", 2, "noul:2", 1.983399510383606),
+ }, shapes
+
+
+def test_page18_drops_the_unconditional_calibration_claim():
+ """The two phrases main uses fire on the ban and are gone here; main has no temp_bucket call."""
+ src = _src18()
+ assert UNQUALIFIED_CALIBRATION_18.search(OLD_PAGE_18), "the ban must fire on main's wording"
+ assert not UNQUALIFIED_CALIBRATION_18.search(src), (
+ "the page still calls `answer_confidence` calibrated without condition")
+ assert not DERIVES_SCALING_18.search(OLD_PAGE_18), "main's page has no scaling lookup"
+ assert not NAMES_CONDITION_18.search(OLD_PAGE_18), "main's page states no calibration condition"
+ assert DERIVES_SCALING_18.search(src), "the page must resolve the bucket via core's temp_bucket"
+ assert NAMES_CONDITION_18.search(src), (
+ "the page must carry the README's Calibration condition (mean ECE 0.466 -> 0.081, "
+ "both shipped checkpoints over-confident)")
+ # The README's exact language lives at line 1586; the page must not misquote it.
+ readme = open(os.path.join(ROOT, "README.md"), encoding="utf-8").read()
+ assert "Both checkpoints are over-confident as shipped" in readme
+ assert "0.466 -> 0.081" in readme
+
+
+for _fn in (test_page18_helpers_are_live_and_defer_to_core,
+ test_page18_questions_reach_core_bucket_names,
+ test_page18_drops_the_unconditional_calibration_claim):
+ try:
+ _fn()
+ except AssertionError as e:
+ FAIL.append("page-18/%s: %s" % (_fn.__name__, e))
+ except Exception as e:
+ FAIL.append("page-18/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
+ else:
+ PASS.append("page-18/%s" % _fn.__name__)
+
+# ------------------------------------------ example 30's score confidence recomputation
+# `examples/30_custom_schema_design.py` teaches the `score` answer type on the page. Main's
+# version asserted -- in prose, with nothing to check it against -- that "`confidence` on a
+# score is normalised entropy, so a wide-but-ordered distribution looks unconfident". That is
+# the inverse of the repo's own definition: `laya.common.confidence_from_probs` returns
+# `1 - H(p) / log(k)`, examples 18/28/40 all describe it as "1 minus normalised entropy", and
+# the sentence was self-contradictory in place -- if confidence were the entropy itself, a wide
+# distribution would read as HIGH confidence, not "unconfident".
+#
+# The fix rewrites the sentence to the correct definition AND recomputes the field on the spot:
+# `entropy_confidence(list(probabilities.values()))` is printed next to
+# `answer["severity"]["confidence"]` for both records, so the reader sees 1 - H/log(k) produce
+# the reported 0.2314 / 0.2019 to within rounding. No weights are loaded here; the gate execs
+# the helper from the example's own AST and drives it against `confidence_from_probs`.
+EXAMPLE_30 = os.path.join(ROOT, "examples", "30_custom_schema_design.py")
+HELPERS_30 = ("entropy_confidence",)
+
+# The page as it ships on main: the sentence this PR replaces, verbatim. The ban below must
+# fire on this text and not on the current example; the positive rules must fire on the
+# current example and not on this text.
+OLD_PAGE_30 = '''
+print(" them, and a choice when the labels are unordered. Read the score itself: `confidence`")
+print(" on a score is normalised entropy, so a wide-but-ordered distribution looks")
+print(" unconfident even when the expected level is informative.")
+'''
+
+# The definition stated as the entropy itself rather than 1 - H/log(k).
+INVERTED_SCORE_DEF = re.compile(
+ r"`confidence`[^.]{0,120}?\bon a score is (?:the )?normali[sz]ed entropy", re.I | re.S)
+# The correct shape, either symbolic or spelled out.
+SCORE_DEFINES_1_MINUS_H = re.compile(
+ r"1\s*-\s*H\s*/\s*log\(k\)|one\s+minus\s+the\s+normali[sz]ed\s+entropy", re.I)
+# The recomputation call must be in the source, not just asserted. Two parts: the page must
+# read the reported probabilities off the answer, and it must feed them into the helper.
+PROBS_READ = re.compile(
+ r"list\(\s*ans\[\"severity\"\]\[\"probabilities\"\]\s*\.\s*values\(\)\s*\)")
+HELPER_CALLED = re.compile(r"entropy_confidence\(")
+# Reported and recomputed values printed side by side (either literal digits or a %.Nf
+# conversion the page formats them with).
+PRINTS_REPORTED_AND_RECOMPUTED = re.compile(
+ r"reported\s+(?:%?\.\d+f|[\d.]+)\s*,\s*recomputed", re.I)
+
+
+def _src30():
+ with open(EXAMPLE_30, encoding="utf-8") as fh:
+ return fh.read()
+
+
+def _helpers30():
+ """Exec the example's pure top-level helpers. `entropy_confidence` reads `math`."""
+ tree = ast.parse(_src30())
+ nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in HELPERS_30]
+ ns = {"math": math}
+ exec(compile(ast.Module(body=nodes, type_ignores=[]), EXAMPLE_30, "exec"), ns) # noqa: S102
+ return ns, {n.name for n in nodes}
+
+
+def _called30():
+ """Names the example's top-level *statements* use, so a helper cannot pass by being dead."""
+ tree = ast.parse(_src30())
+ top = [n for n in tree.body if not isinstance(
+ n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Import, ast.ImportFrom))]
+ return {node.id for stmt in top for node in ast.walk(stmt)
+ if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)}
+
+
+def test_page30_helpers_are_live_and_pure():
+ """`entropy_confidence` is defined, called at top level, and returns rather than prints."""
+ _, defined = _helpers30()
+ assert defined == set(HELPERS_30), "example 30 lost a helper: %s" % sorted(defined)
+ unused = set(HELPERS_30) - _called30()
+ assert not unused, "defined but never called at module level: %s" % sorted(unused)
+ tree = ast.parse(_src30())
+ for name in HELPERS_30:
+ node = next(n for n in tree.body
+ if isinstance(n, ast.FunctionDef) and n.name == name)
+ prints = [c for c in ast.walk(node)
+ if isinstance(c, ast.Call) and isinstance(c.func, ast.Name)
+ and c.func.id == "print"]
+ assert not prints, "%s prints instead of returning" % name
+
+
+def test_entropy_confidence_matches_core():
+ """Recomputing 1 - H/log(k) by hand must equal `confidence_from_probs` on the same list."""
+ ns = _helpers30()[0]
+ for probs in ([0.5, 0.5], [0.1, 0.9], [0.25, 0.25, 0.25, 0.25],
+ [0.7, 0.2, 0.05, 0.05], [1e-9, 0.999999999, 0.0],
+ [0.4, 0.3, 0.2, 0.1], [0.05, 0.15, 0.3, 0.35, 0.1, 0.05]):
+ got = ns["entropy_confidence"](list(probs))
+ want = confidence_from_probs(np.array(probs, dtype=float), len(probs))
+ assert abs(got - want) < 1e-9, (probs, got, want)
+ assert ns["entropy_confidence"]([1.0]) == 1.0, "k=1 is certain"
+ assert ns["entropy_confidence"]([]) == 1.0, "k=0 does not crash"
+
+
+def test_page30_drops_the_inverted_definition():
+ """The ban fires on main's sentence and not on this one; the positive rule inverts."""
+ src = _src30()
+ assert INVERTED_SCORE_DEF.search(OLD_PAGE_30), \
+ "the ban does not fire on main's wording"
+ assert not INVERTED_SCORE_DEF.search(src), \
+ "the inverted definition is still in example 30"
+ assert SCORE_DEFINES_1_MINUS_H.search(src), \
+ "the correct 1 - H/log(k) definition is missing"
+ assert not SCORE_DEFINES_1_MINUS_H.search(OLD_PAGE_30), \
+ "the positive rule passes on main's page too"
+
+
+def test_page30_recomputes_in_place():
+ """The reader sees the field cross-checked against the probabilities that produced it."""
+ src = _src30()
+ assert PROBS_READ.search(src), \
+ "the score section no longer reads the reported `probabilities` list"
+ assert HELPER_CALLED.search(src), \
+ "the score section no longer calls `entropy_confidence`"
+ assert PRINTS_REPORTED_AND_RECOMPUTED.search(src), \
+ "reported and recomputed values are not printed side by side"
+ assert not PROBS_READ.search(OLD_PAGE_30)
+ assert not HELPER_CALLED.search(OLD_PAGE_30)
+ assert not PRINTS_REPORTED_AND_RECOMPUTED.search(OLD_PAGE_30)
+
+
+for _fn in (test_page30_helpers_are_live_and_pure,
+ test_entropy_confidence_matches_core,
+ test_page30_drops_the_inverted_definition,
+ test_page30_recomputes_in_place):
+ try:
+ _fn()
+ except AssertionError as e:
+ FAIL.append("page-30/%s: %s" % (_fn.__name__, e))
+ except Exception as e:
+ FAIL.append("page-30/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
+ else:
+ PASS.append("page-30/%s" % _fn.__name__)
+
+
+# -------------------------------- evals' --min-confidence wording must match the gate's effect
+#
+# `laya/evals_cli.py`, `laya/evals.py`, `docs/evals.md`, `tests/test_evals.py` and
+# `tests/test_evals_api.py` each describe the abstention gate's role in a scored run. Their
+# previous wording claimed the threshold changes what scores: "answers below it come back
+# abstained", "an abstention overwrites a low-confidence choice", "the run scores the policy
+# at that threshold, not the raw argmax", "a `precision@coverage` figure for a policy that
+# never ran". But `apply_confidence_gate` in this module writes `low_confidence` and
+# `abstention` state fields and leaves `answer["choice"]/["noul"]/["score"]` untouched, and
+# every `Evaluator.score` and `_correct` in `laya/evals.py` reads only those value keys. So the
+# metrics -- accuracy, ece, brier, aurc, selective_accuracy@NN -- are identical at every
+# threshold; what changes is `report.config["timing"]["min_confidence"]` and `min_confidence_sent`,
+# which the wording below must describe as claims about the run, not claims about the answers.
+
+_EVALS_CLI_PATH = os.path.join(ROOT, "laya", "evals_cli.py")
+_EVALS_PY_PATH = os.path.join(ROOT, "laya", "evals.py")
+_EVALS_MD_PATH = os.path.join(ROOT, "docs", "evals.md")
+_TEST_EVALS_PATH = os.path.join(ROOT, "tests", "test_evals.py")
+_TEST_EVALS_API_PATH = os.path.join(ROOT, "tests", "test_evals_api.py")
+
+
+def _read_text(path):
+ with open(path, "r", encoding="utf-8") as f:
+ return f.read()
+
+
+def _docstring_of(path, func_name):
+ """A function's docstring pulled out of the file, so a ban can point at one site not the file."""
+ tree = ast.parse(_read_text(path))
+ for node in ast.walk(tree):
+ if isinstance(node, ast.FunctionDef) and node.name == func_name:
+ return ast.get_docstring(node) or ""
+ return ""
+
+
+def _cli_help_of(path, flag):
+ """The concatenated `help=` literal on a `parser.add_argument(flag, ...)` call."""
+ tree = ast.parse(_read_text(path))
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Call) and node.args \
+ and isinstance(node.args[0], ast.Constant) and node.args[0].value == flag:
+ for kw in node.keywords:
+ if kw.arg == "help":
+ try:
+ return ast.literal_eval(kw.value) or ""
+ except (ValueError, SyntaxError):
+ return ""
+ return ""
+
+
+# The wrong-language patterns: each is a claim that the threshold changes the scored numbers.
+_BAN_COME_BACK_ABSTAINED = re.compile(r"come back\s+abstained", re.I)
+_BAN_OVERWRITES_A_LOW = re.compile(r"abstention\s+overwrites\s+a\s+low", re.I)
+_BAN_SCORES_POLICY = re.compile(r"scores the policy at (?:the|that) threshold", re.I)
+_BAN_VS_RAW_ARGMAX = re.compile(r"(?:rather than|not)\s+the\s+raw\s+argmax", re.I)
+_BAN_MINCONFIDENCE_CHANGES_ANSWER = re.compile(
+ r"`min_confidence`[^.\n]{0,80}?changes the answ", re.I)
+_BAN_UNLIKE_GROUPING_CHANGES = re.compile(
+ r"[Uu]nlike\s+`?grouping`?[^.]{0,40}?changes the answ", re.I)
+_BAN_PRECISION_NUMBER_FOR_POLICY = re.compile(
+ r"precision@coverage`?\s+(?:number|figure)\s+for a policy that never ran", re.I)
+_BAN_WOULD_REPORT_PRECISION = re.compile(
+ r"would report `?precision@coverage`?", re.I)
+_BAN_OTHERWISE_SCORE_POLICY = re.compile(
+ r"otherwise score a policy that never ran", re.I)
+_BAN_NO_WAY_TO_MEASURE_PATCOV = re.compile(
+ r"no way to measure `?precision@coverage", re.I)
+_BAN_A_SCORING_CONTROL = re.compile(r"(?:it is|that is|dropping) a\b[^.\n]{0,10}scoring control", re.I)
+_BAN_ANSWER_NOT_SAME_DECISION = re.compile(
+ r"an answer below the threshold is not the same decision", re.I)
+_BAN_T0_T07_DIFFERENT_EXPERIMENT = re.compile(
+ r"same run at `T=0` and `T=0\.7` is a different experiment", re.I)
+_BAN_SWEEP_IS_SERIES = re.compile(
+ r"`precision@coverage`\s+sweep is a series of these", re.I)
+_BAN_CHANGES_WHICH_ANSWERS_SCORE = re.compile(
+ r"(?:abstention threshold|min_confidence).{0,30}?changes which answers score", re.I | re.S)
+
+_EVALS_BANS = (
+ _BAN_COME_BACK_ABSTAINED, _BAN_OVERWRITES_A_LOW, _BAN_SCORES_POLICY,
+ _BAN_VS_RAW_ARGMAX, _BAN_MINCONFIDENCE_CHANGES_ANSWER,
+ _BAN_UNLIKE_GROUPING_CHANGES, _BAN_PRECISION_NUMBER_FOR_POLICY,
+ _BAN_WOULD_REPORT_PRECISION, _BAN_OTHERWISE_SCORE_POLICY,
+ _BAN_NO_WAY_TO_MEASURE_PATCOV, _BAN_A_SCORING_CONTROL,
+ _BAN_ANSWER_NOT_SAME_DECISION, _BAN_T0_T07_DIFFERENT_EXPERIMENT,
+ _BAN_SWEEP_IS_SERIES, _BAN_CHANGES_WHICH_ANSWERS_SCORE,
+)
+
+# Positive claims the wording must make: name the two state fields `apply_confidence_gate`
+# writes. A wording that only says what the gate does NOT do is a diff, not a fix.
+_SAYS_LOW_CONFIDENCE = re.compile(r"low_confidence")
+_SAYS_ABSTENTION_FIELD = re.compile(
+ r"abstention[^a-zA-Z]{0,3}[:=]?[^a-zA-Z]{0,3}[\"']abstained[^a-zA-Z]{0,3}[\"']")
+
+# The pre-fix wording at each of the ten sites we touched, so every ban above has a witness
+# it fires on. A ban that never fires is a rule the code cannot check.
+_OLD_E1 = ("abstention threshold on `answer_confidence` (#361): answers below it come back "
+ "abstained, so the run scores the policy at that threshold rather than the raw "
+ "argmax.")
+_OLD_E2 = ("`min_confidence` changes the answer (an abstention overwrites a low-confidence "
+ "choice), so it is a scoring control, not an optimisation. silently dropping the "
+ "threshold and reporting the same run would give a `precision@coverage` number for "
+ "a policy that never ran")
+_OLD_E3 = ("answers below it come back abstained, so the run scores the policy at that "
+ "threshold, not the raw argmax. Unlike `sort_by_length` this changes the answers")
+_OLD_E4 = ("because an abstention threshold changes which answers score as correct. Silently "
+ "dropping it would publish a `precision@coverage` number for a policy that never ran")
+_OLD_E5 = ("or drop the threshold -- the report would otherwise score a policy that never ran")
+_OLD_MD1 = ("Unlike grouping, this changes the answers that score: the same run at `T=0` and "
+ "`T=0.7` is a different experiment, and a `precision@coverage` sweep is a series "
+ "of these, not a single baseline drifting.")
+_OLD_MD2 = ("Silently dropping a scoring control is the class of lie this harness exists to "
+ "prevent: the report would publish a `precision@coverage` figure for a policy "
+ "that never ran.")
+_OLD_T1 = ("That is a scoring control: an answer below the threshold is not the same decision "
+ "as one above. A `laya-evals run` that could not pass it through had no way to "
+ "measure `precision@coverage` at any threshold")
+_OLD_T2 = ("Silently dropping a scoring control would report `precision@coverage` for a "
+ "policy that never ran.")
+_OLD_T3 = ("a `--min-confidence` run that silently dropped the argument would publish a "
+ "`precision@coverage` figure for a policy that never ran")
+
+_OLD_EVALS_SITES = (_OLD_E1, _OLD_E2, _OLD_E3, _OLD_E4, _OLD_E5,
+ _OLD_MD1, _OLD_MD2, _OLD_T1, _OLD_T2, _OLD_T3)
+
+
+def test_abstention_wording_bans_have_teeth():
+ """Every ban fires on a pre-fix site, and every pre-fix site is caught by at least one ban."""
+ matched = [[bool(rule.search(old)) for rule in _EVALS_BANS] for old in _OLD_EVALS_SITES]
+ for i, old in enumerate(_OLD_EVALS_SITES):
+ assert any(matched[i]), "site %d's pre-fix wording is not caught by any ban:\n%s" % (i, old)
+ for j, rule in enumerate(_EVALS_BANS):
+ assert any(matched[i][j] for i in range(len(_OLD_EVALS_SITES))), \
+ "ban /%s/ never fires on any pre-fix wording: a rule without a witness" % rule.pattern
+
+
+def test_abstention_wording_is_gone_from_every_site():
+ """None of the scoring-change claims may appear in the sources they were removed from."""
+ for path in (_EVALS_CLI_PATH, _EVALS_PY_PATH, _EVALS_MD_PATH,
+ _TEST_EVALS_PATH, _TEST_EVALS_API_PATH):
+ src = _read_text(path)
+ for rule in _EVALS_BANS:
+ assert not rule.search(src), \
+ "%s still carries the scoring-change claim: /%s/" % (path, rule.pattern)
+
+
+def test_abstention_wording_names_the_state_fields():
+ """Each site individually must name what the gate writes, not just somewhere in the file.
+
+ Per-docstring rather than per-file, so a mutation that drops `low_confidence` from the
+ `evaluate()` docstring alone is caught even though the sibling `_takes_min_confidence`
+ docstring above it still names the field.
+ """
+ help_text = _cli_help_of(_EVALS_CLI_PATH, "--min-confidence")
+ assert help_text, "no `help=` string on the --min-confidence argument in laya/evals_cli.py"
+ assert _SAYS_LOW_CONFIDENCE.search(help_text), \
+ "evals_cli's --min-confidence help does not name the `low_confidence` field"
+ assert _SAYS_ABSTENTION_FIELD.search(help_text), \
+ "evals_cli's --min-confidence help does not name `abstention: \"abstained\"`"
+
+ for func_name in ("_takes_min_confidence", "evaluate"):
+ doc = _docstring_of(_EVALS_PY_PATH, func_name)
+ assert doc, "no docstring on laya/evals.py::%s" % func_name
+ assert _SAYS_LOW_CONFIDENCE.search(doc), \
+ "laya/evals.py::%s docstring does not name the `low_confidence` field" % func_name
+ assert _SAYS_ABSTENTION_FIELD.search(doc), \
+ "laya/evals.py::%s docstring does not name `abstention: \"abstained\"`" % func_name
+
+ md = _read_text(_EVALS_MD_PATH)
+ assert _SAYS_LOW_CONFIDENCE.search(md), \
+ "docs/evals.md does not name the `low_confidence` field"
+ assert _SAYS_ABSTENTION_FIELD.search(md), \
+ "docs/evals.md does not name `abstention: \"abstained\"`"
+
+
+def test_gate_writes_the_fields_the_docs_name():
+ """The witness that ties the wording above to `apply_confidence_gate`'s actual effect.
+
+ Drive the gate at 0.7 over one high-confidence and one low-confidence `choice` answer, both
+ carrying `choice: "keep"`. The low one must come back flagged; both must keep their raw
+ argmax in `answer["choice"]`. If the gate ever starts overwriting the value, this assertion
+ breaks and the wording claim breaks with it.
+ """
+ results = [{"answers": {
+ "q_high": {"type": "choice", "choice": "keep",
+ "answer_confidence": 0.90,
+ "probabilities": {"keep": 0.90, "drop": 0.10}},
+ "q_low": {"type": "choice", "choice": "keep",
+ "answer_confidence": 0.40,
+ "probabilities": {"keep": 0.40, "drop": 0.60}},
+ }}]
+ apply_confidence_gate(results, 0.7)
+ high = results[0]["answers"]["q_high"]
+ low = results[0]["answers"]["q_low"]
+ check("evals-wording/high clears the gate", high["abstention"], GATE_PASSED)
+ check("evals-wording/low falls below", low["abstention"], GATE_ABSTAINED)
+ check_true("evals-wording/low is flagged", low.get("low_confidence") is True)
+ check_true("evals-wording/high is not flagged", "low_confidence" not in high)
+ # The claim the wording makes about the gate: the value key stays at the raw argmax.
+ check("evals-wording/high's choice is unchanged", high["choice"], "keep")
+ check("evals-wording/low's choice is unchanged even when abstained", low["choice"], "keep")
+ # Both thresholds echoed onto the answer, so `abstention_threshold` matches the wording.
+ check("evals-wording/high echoes the threshold", high["abstention_threshold"], 0.7)
+ check("evals-wording/low echoes the threshold", low["abstention_threshold"], 0.7)
+
+
+for _fn in (test_abstention_wording_bans_have_teeth,
+ test_abstention_wording_is_gone_from_every_site,
+ test_abstention_wording_names_the_state_fields,
+ test_gate_writes_the_fields_the_docs_name):
+ try:
+ _fn()
+ except AssertionError as e:
+ FAIL.append("evals-abstention/%s: %s" % (_fn.__name__, e))
+ except Exception as e:
+ FAIL.append("evals-abstention/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
+ else:
+ PASS.append("evals-abstention/%s" % _fn.__name__)
+
+# ------------------------------------------ example 26's banner must count the primitives triage_questions() returns
+# `examples/26_presets_triage.py` opened with "`triage_questions()` is a ready-made schema for
+# inbound support: five questions ... a multi-way choice, two yes/no probabilities, and an ordinal
+# score". `laya.triage_questions()` ships three `noul` fields -- `is_urgent`, `refund_requested`,
+# `churn_risk` -- plus the choice and the score, so the banner's own arithmetic read 1 + 2 + 1 = 4
+# against its stated "five questions", and the example's own print block already prints all three
+# `noul` values two paragraphs later. A reader counting primitives against the schema could not
+# reconcile them.
+EXAMPLE_26 = os.path.join(ROOT, "examples", "26_presets_triage.py")
+OLD_BANNER_26 = '''
+ `triage_questions()` is a ready-made schema for inbound support: five questions, one
+ forward pass. It mixes all three primitives -- a multi-way choice, two yes/no
+ probabilities, and an ordinal score -- so a single call fills a whole triage record.
+'''
+_BAN_TWO_YESNO = re.compile(r"\btwo\s+yes/no\s+probabilit", re.I)
+# The banner's own stated counts. `_WORDS` maps each spelling that could stand for one primitive.
+_STATED_TOTAL_26 = re.compile(r"\b(one|two|three|four|five|six)\s+questions", re.I)
+_STATED_YESNO_26 = re.compile(r"\b(one|two|three|four|five|six)\s+yes/no\s+probabilit", re.I)
+_STATED_CHOICE_26 = re.compile(r"\b(a|one)\s+multi-way\s+choice", re.I)
+_STATED_SCORE_26 = re.compile(r"\b(a|an)\s+ordinal\s+score", re.I)
+_WORDS = {"one": 1, "a": 1, "an": 1, "two": 2, "three": 3, "four": 4, "five": 5, "six": 6}
+
+
+def _src26():
+ with open(EXAMPLE_26, encoding="utf-8") as fh:
+ return fh.read()
+
+
+def _triage_type_counts():
+ from laya.presets import triage_questions
+ q = triage_questions()
+ by_type = {}
+ for spec in q.values():
+ by_type[spec["type"]] = by_type.get(spec["type"], 0) + 1
+ return len(q), by_type
+
+
+def test_triage_preset_is_one_choice_three_noul_one_score():
+ # The gate's positive anchor: this is the schema the banner must describe. If `triage_questions()`
+ # ever moves, the banner test fails here first, and the drift is loud rather than silent.
+ n_total, by_type = _triage_type_counts()
+ assert n_total == 5, "triage_questions() no longer returns five fields: %r" % (by_type,)
+ assert by_type == {"choice": 1, "noul": 3, "score": 1}, \
+ "triage_questions()'s type mix moved: %r" % (by_type,)
+
+
+def test_example_26_banner_counts_match_the_preset():
+ n_total, by_type = _triage_type_counts()
+ src = _src26()
+ m_total = _STATED_TOTAL_26.search(src)
+ m_yesno = _STATED_YESNO_26.search(src)
+ m_choice = _STATED_CHOICE_26.search(src)
+ m_score = _STATED_SCORE_26.search(src)
+ assert m_total and m_yesno and m_choice and m_score, (
+ "the banner no longer spells out its primitive counts; the gate needs them readable")
+ assert _WORDS[m_total.group(1).lower()] == n_total, \
+ "banner says %r questions, triage_questions() returns %d" % (m_total.group(1), n_total)
+ assert _WORDS[m_yesno.group(1).lower()] == by_type["noul"], \
+ "banner says %r yes/no probabilities, triage_questions() returns %d noul fields" % \
+ (m_yesno.group(1), by_type["noul"])
+ assert _WORDS[m_choice.group(1).lower()] == by_type["choice"]
+ assert _WORDS[m_score.group(1).lower()] == by_type["score"]
+ # Parts must add up to the whole the same paragraph asserts.
+ parts = (_WORDS[m_choice.group(1).lower()] + _WORDS[m_yesno.group(1).lower()]
+ + _WORDS[m_score.group(1).lower()])
+ assert parts == _WORDS[m_total.group(1).lower()], (
+ "banner arithmetic is wrong: %d + %d + %d = %d but it says %d questions" % (
+ _WORDS[m_choice.group(1).lower()], _WORDS[m_yesno.group(1).lower()],
+ _WORDS[m_score.group(1).lower()], parts, _WORDS[m_total.group(1).lower()]))
+
+
+def test_example_26_drops_the_wrong_yes_no_count():
+ # The ban has teeth: it fires on the pre-fix wording shipped on main.
+ assert _BAN_TWO_YESNO.search(OLD_BANNER_26), "the ban must fire on the pre-fix wording"
+ # ...and every count regex still reads the OLD banner as its pre-fix numbers, so the positive
+ # rule was genuinely falsifiable.
+ m = _STATED_YESNO_26.search(OLD_BANNER_26)
+ assert m is not None and _WORDS[m.group(1).lower()] == 2
+ # The current example satisfies the ban.
+ assert not _BAN_TWO_YESNO.search(_src26()), "the wrong yes/no count is back in the shipped example"
+
+
+for _fn in (test_triage_preset_is_one_choice_three_noul_one_score,
+ test_example_26_banner_counts_match_the_preset,
+ test_example_26_drops_the_wrong_yes_no_count):
+ try:
+ _fn()
+ except AssertionError as e:
+ FAIL.append("page-26/%s: %s" % (_fn.__name__, e))
+ except Exception as e: # a crash is a failure, never a silent pass
+ FAIL.append("page-26/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
+ else:
+ PASS.append("page-26/%s" % _fn.__name__)
+
+ # ------------------------------------------- tl_kernels' docstring must name the caller's real padding policy
+ # `laya/tl_kernels.py`'s module docstring said "M must be a multiple of 16 (the caller pads);
+ # out-of-bounds rows are predicated by TileLang". The caller pads two dimensions, not one:
+ # `laya/backends/base.py::bucket_rows` pads the batch dim to the next power of two (1, 2, 4, 8,
+ # 16, 32, ...) and `bucket_tokens` pads the sequence dim to a 16-token bucket up to
+ # `DYNAMIC_MAX_L` and a 64-token bucket beyond. M is the flattened `rows * tokens`, so the
+ # multiple-of-16 came from the token dim, never from the row dim. A reader who took the
+ # docstring as a row constraint would add `assert M % 16 == 0` at the call site and break
+ # every batch of 1, 2, 4, or 8 questions -- and the shipped `pad_batch` tests already pin those
+ # shapes: `tests/test_backends.py:86` "pad_batch/one row stays one row" is (1, 16), and
+ # `:89` "pad_batch/five rows pad to eight" is (8, 16).
+ TL_KERNELS = os.path.join(ROOT, "laya", "tl_kernels.py")
+ OLD_TL_DOCSTRING = (
+ "GPU kernels take 16-bit activations (bf16 by default, fp16 with dtype=\"float16\"), accumulate in fp32. Row count M is a runtime\n"
+ "symbol so one compiled kernel serves every batch/sequence bucket; M must be a\n"
+ "multiple of 16 (the caller pads); out-of-bounds rows are predicated by TileLang.\n"
+ "Use compile_cpu for an explicit fp32 CPU specialization; the GPU defaults are unchanged."
+ )
+ _BAN_ROW_MULT_16 = re.compile(r"M must be a\s*\n?\s*multiple of 16 \(the caller pads\)", re.I)
+ _BAN_MULT_16_FROM_ROW = re.compile(r"multiple[- ]of[- ]16[^.\n]{0,80}from the row dim", re.I)
+ _NAMED_POWER_OF_TWO = re.compile(r"power of two", re.I)
+ _NAMED_16_TOKEN_BUCKET = re.compile(r"16[- ]token bucket", re.I)
+ _NAMED_NOT_ROW_DIM = re.compile(r"not the row dim", re.I)
+ _NAMED_BUCKET_ROWS = re.compile(r"\bbucket_rows\b")
+ _NAMED_BUCKET_TOKENS = re.compile(r"\bbucket_tokens\b")
+
+
+ def _tl_docstring():
+ """Return `laya/tl_kernels.py`'s module docstring via `ast`, so the gate reads what a
+ reader reads -- not whatever happens to appear in a comment."""
+ with open(TL_KERNELS, encoding="utf-8") as fh:
+ src = fh.read()
+ return ast.get_docstring(ast.parse(src)) or ""
+
+
+ def test_tl_kernels_drops_the_wrong_row_claim():
+ # The ban fires on the pre-fix docstring...
+ assert _BAN_ROW_MULT_16.search(OLD_TL_DOCSTRING), "the ban must fire on the pre-fix wording"
+ # ...and does not fire on the shipped one.
+ doc = _tl_docstring()
+ assert doc, "the tl_kernels module docstring was removed -- the gate has nothing to read"
+ assert not _BAN_ROW_MULT_16.search(doc), (
+ "`laya/tl_kernels.py` again says M must be a multiple of 16 because the caller pads; "
+ "the caller pads rows to a power of two (1, 2, 4, 8, ...). A reader who trusts the "
+ "claim asserts `M % 16 == 0` at the call site and breaks every small batch.")
+ # The second ban catches the mis-attribution (claiming the 16-multiple comes from rows).
+ assert not _BAN_MULT_16_FROM_ROW.search(doc), (
+ "the docstring attributes the multiple-of-16 to the row dim; `bucket_rows` pads to a "
+ "power of two, so the 16-multiple is a token-dim artefact.")
+
+
+ def test_tl_kernels_names_the_real_padding_policy():
+ doc = _tl_docstring()
+ assert _NAMED_POWER_OF_TWO.search(doc), (
+ "the docstring must name the row pad as a power of two, matching `bucket_rows`")
+ assert _NAMED_16_TOKEN_BUCKET.search(doc), (
+ "the docstring must name the token pad as a 16-token bucket, matching `bucket_tokens`")
+ assert _NAMED_BUCKET_ROWS.search(doc), (
+ "the docstring must attribute the row pad to `bucket_rows`, the function that does it")
+ assert _NAMED_BUCKET_TOKENS.search(doc), (
+ "the docstring must attribute the token pad to `bucket_tokens`, the function that does it")
+ assert _NAMED_NOT_ROW_DIM.search(doc), (
+ "the docstring must name the row dim as the wrong source of the 16-multiple, so a "
+ "reader cannot re-attribute it back")
+
+
+ def test_bucket_rows_stays_a_power_of_two_below_16():
+ # The witness: rows below 16 really do stay at 1, 2, 4, 8, so the pre-fix claim
+ # ("M must be a multiple of 16 (the caller pads)") was false for them. If `bucket_rows`
+ # ever starts rounding to 16, the docstring fix needs review and this test says so.
+ from laya.backends.base import bucket_rows
+ for n, want in ((1, 1), (2, 2), (3, 4), (4, 4), (5, 8), (7, 8), (8, 8), (9, 16),
+ (15, 16), (17, 32), (24, 32), (33, 64)):
+ got = bucket_rows(n)
+ assert got == want, (
+ "bucket_rows(%d) returned %d, expected %d -- the docstring's row contract needs "
+ "review" % (n, got, want))
+ assert got & (got - 1) == 0, (
+ "bucket_rows(%d) = %d is not a power of two; the docstring's `power of two` claim "
+ "is stale" % (n, got))
+ # Rows below 16 must stay below 16 -- that is the case the pre-fix docstring got wrong.
+ for n in (1, 2, 4, 8):
+ assert bucket_rows(n) == n, (
+ "bucket_rows(%d) = %d; a caller that trusts the old multiple-of-16 claim would "
+ "expect 16 here" % (n, bucket_rows(n)))
+
+
+ for _fn in (test_tl_kernels_drops_the_wrong_row_claim,
+ test_tl_kernels_names_the_real_padding_policy,
+ test_bucket_rows_stays_a_power_of_two_below_16):
+ try:
+ _fn()
+ except AssertionError as e:
+ FAIL.append("tl-kernels/%s: %s" % (_fn.__name__, e))
+ except Exception as e: # a crash is a failure, never a silent pass
+ FAIL.append("tl-kernels/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
+ else:
+ PASS.append("tl-kernels/%s" % _fn.__name__)
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL", f)