Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

test(state-budget): hold the high-cardinality page to build_head's thresholds and floors

Aashish7474bc3c9215
diff --git a/tests/test_state_budget.py b/tests/test_state_budget.py
index 65f0f198..d35d4ed3 100644
--- a/tests/test_state_budget.py
+++ b/tests/test_state_budget.py
@@ -16,14 +16,25 @@ from the head it actually built.
This suite pins the relationship, and — when the checkpoints are present — reads the figures
back out of `README.md` and checks them against a live measurement, so the renderer, the
documentation and this file cannot drift apart silently.
+
+`examples/33_high_cardinality_choice.py` is the page that teaches this budget, and it taught three
+things the code does not do: that the cut waits for the options to *overflow* `head_max_len` (it
+fires on the spare budget, while they still fit), that 16 is a minimum *reserved for the instruction*
+(the instruction is held to 8), and that the option block *lives in* `head_max_len` (past ~4 tokens
+per option the head grows past the cap and takes the state's room with it). The last sections of
+this file pin the page to `build_head` — behaviourally, and by reading the budget's literals out of
+the function instead of typing them in here.
"""
+import ast
+import inspect
import os
import re
import sys
+import textwrap
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-from laya.common import build_sequence # noqa: E402
+from laya.common import build_head, build_sequence, render_options # noqa: E402
PASS, FAIL, SKIPPED = [], [], []
@@ -214,12 +225,253 @@ def test_readme_quotes_the_measured_figures():
skip("no checkpoints under %s" % _models_dir())
+# ---------------------------------------------------- example 33: the page against the budget it teaches
+
+EXAMPLE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+ "examples", "33_high_cardinality_choice.py")
+
+# The two formulas, in the symbolic form the page has to teach them in.
+PER_FORMULA = re.compile(r"per\s*=\s*max\(\s*(\d+)\s*,\s*\(\s*head_max_len\s*-\s*(\d+)\s*\)"
+ r"\s*//\s*N\s*\)")
+INSTRUCTION_FORMULA = re.compile(r"head_ids\[:\s*max\(\s*(\d+)\s*,\s*opt_budget\s*\)\]")
+# `max(4, (192-16)//77) = 4` -- a worked instance, whose arithmetic has to come out right.
+WORKED_INSTANCE = re.compile(r"max\(\s*(\d+)\s*,\s*\((\d+)\s*-\s*(\d+)\)\s*//\s*(\d+)\s*\)"
+ r"\s*=\s*(\d+)")
+CONDITIONAL = re.compile(r"\b(?:when|once|if|unless)\b[^,;.]{0,80}", re.I)
+OVERFLOW = re.compile(r"overflow", re.I)
+# 16 taught as the instruction's reserve: the number that is really the instruction's floor is 8.
+RESERVE_16_FOR_INSTRUCTION = re.compile(
+ r"\b16\b[^.]{0,60}\b(?:reserv\w*|minimum)\b[^.]{0,60}instruction"
+ r"|instruction[^.]{0,60}\b(?:reserv\w*|minimum)\b[^.]{0,60}\b16\b", re.I)
+BLOCK_INSIDE_CAP = re.compile(
+ r"\b(?:lives|live|fits|fit|stays|stay|kept)\b[^.]{0,30}\b(?:in|within|under|inside)\b[^.]{0,30}"
+ r"head_max_len", re.I)
+CAP_IS_NOT_HARD = re.compile(r"not a hard cap|no hard cap|past the cap|past head_max_len"
+ r"|longer than head_max_len|head longer than|grows past", re.I)
+INSTRUCTION_FLOOR_8 = re.compile(r"max\(\s*8\s*,|8-token|\b8\b[^.]{0,40}instruction"
+ r"|instruction[^.]{0,40}\b8\b", re.I)
+
+
+def _page_prose():
+ """The example's string constants, whitespace-normalised, as (joined, sentences).
+
+ Sentences are split inside each constant, never over a join of them: two adjacent constants
+ can read as one sentence, which would let a true sentence vouch for a false neighbour.
+ """
+ if not os.path.exists(EXAMPLE):
+ raise AssertionError("examples/33_high_cardinality_choice.py is not where this suite looks "
+ "for it (%s)" % EXAMPLE)
+ with open(EXAMPLE, encoding="utf-8") as fh:
+ tree = ast.parse(fh.read())
+ constants = [re.sub(r"\s+", " ", node.value) for node in ast.walk(tree)
+ if isinstance(node, ast.Constant) and isinstance(node.value, str)]
+ sentences = [sent.strip() for text in constants
+ for sent in re.split(r"(?<=[.!?])\s+", text) if sent.strip()]
+ return " ".join(constants), sentences
+
+
+def _option_question(lengths, ins="Which one?"):
+ """A choice question whose options cost exactly `lengths` tokens under `_FakeTok`.
+
+ One id per word, and an option renders as `label: <criterion>`, so the criterion carries
+ `length - 2` words: the label and its colon are the other two.
+ """
+ return {"t": "choice", "ins": ins,
+ "crit": {"o%d" % i: " ".join(["w"] * (length - 2)) for i, length in enumerate(lengths)}}
+
+
+def _full_option_len(tok, q):
+ """Tokens each option costs before the budget cuts it, `[MASK]` included."""
+ return [1 + len(tok(" " + opt)["input_ids"]) for opt in render_options(q)]
+
+
+def _budget_literals():
+ """(threshold, reserve, per-floor, instruction-floor) exactly as `build_head` writes them.
+
+ Read out of the function rather than typed in here, so a page holding a stale number fails
+ instead of agreeing with itself.
+ """
+ tree = ast.parse(textwrap.dedent(inspect.getsource(build_head)))
+ found = {}
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Compare) and len(node.ops) == 1 and isinstance(node.ops[0], ast.Lt):
+ left, right = node.left, node.comparators[0]
+ if isinstance(left, ast.Name) and left.id == "opt_budget" and isinstance(right, ast.Constant):
+ found["threshold"] = right.value
+ if isinstance(node, ast.Call) and getattr(node.func, "id", "") == "max" and len(node.args) == 2:
+ first, second = node.args
+ if not isinstance(first, ast.Constant):
+ continue
+ if isinstance(second, ast.BinOp) and isinstance(second.op, ast.FloorDiv) \
+ and isinstance(second.left, ast.BinOp) and isinstance(second.left.op, ast.Sub) \
+ and getattr(second.left.left, "id", "") == "head_max_len" \
+ and isinstance(second.left.right, ast.Constant):
+ found["per_floor"] = first.value
+ found["reserve"] = second.left.right.value
+ if isinstance(second, ast.Name) and second.id == "opt_budget":
+ found["instruction_floor"] = first.value
+ missing = {"threshold", "reserve", "per_floor", "instruction_floor"} - set(found)
+ assert not missing, (
+ "build_head no longer writes the budget the way this suite reads it: %s missing"
+ % sorted(missing))
+ return found
+
+
+def test_the_option_cut_waits_on_spare_tokens_not_on_overflow():
+ """The cut fires on the leftover head budget, and fires while the options still fit.
+
+ This is the ground truth the page has to match: `head_max_len - sum(options) < 16`, not the
+ options overflowing `head_max_len`. A question whose options fill the head exactly -- 192 of
+ 192, nothing dropped -- is already cut.
+ """
+ tok = _FakeTok()
+ head_max_len = 192
+
+ fills_it = _option_question([48, 48, 48, 48])
+ full = _full_option_len(tok, fills_it)
+ check("fixture: four options of 48 fill the head without overflowing it",
+ (sum(full), sum(full) > head_max_len, head_max_len - sum(full)), (192, False, 0))
+ _, _, stats = build_head(tok, fills_it, head_max_len)
+ assert stats["tokens_per_option"] is not None, (
+ "a question that leaves 0 spare tokens must already be cut to "
+ "max(4, (head_max_len - 16) // N); it kept its full lengths, so the cut is not where "
+ "this suite says it is")
+ check("cut size for 4 options at head_max_len=192", stats["tokens_per_option"], 44)
+
+ spare_16 = _option_question([44, 44, 44, 44])
+ check("fixture: 16 spare tokens", head_max_len - sum(_full_option_len(tok, spare_16)), 16)
+ _, _, stats_16 = build_head(tok, spare_16, head_max_len)
+ check("16 spare tokens is not yet the cut", stats_16["tokens_per_option"], None)
+
+ spare_15 = _option_question([45, 44, 44, 44])
+ check("fixture: 15 spare tokens", head_max_len - sum(_full_option_len(tok, spare_15)), 15)
+ _, _, stats_15 = build_head(tok, spare_15, head_max_len)
+ assert stats_15["tokens_per_option"] is not None, (
+ "15 spare tokens is under the 16 the cut waits for, so this must be cut")
+
+
+def test_the_instruction_floor_is_eight_not_the_reserved_sixteen():
+ """Past the floor the instruction keeps 8 tokens -- 16 is not reserved for it.
+
+ 77 options at the 4-token floor leave the head short of itself, and the instruction is clipped
+ to `max(8, opt_budget)` with a negative `opt_budget`: the retained prefix is 8, from a question
+ whose instruction is far longer.
+ """
+ tok = _FakeTok()
+ ins = ("Which of these many clearly distinct banking intents best matches the customer "
+ "message right now, please answer carefully")
+ q = _many_options(77)
+ q["ins"] = ins
+ full_instruction = len(tok("%s question: %s" % (q["t"], ins))["input_ids"])
+ assert full_instruction > 16, (
+ "fixture needs an instruction long enough that a 16-token reserve would show up")
+
+ ids, markers, stats = build_head(tok, q, 192)
+ check("77 options are cut to the per-option floor", stats["tokens_per_option"], 4)
+ check("instruction tokens retained", markers[0] - 2, 8)
+ check("the retained prefix is below the 16 the page must not call its reserve",
+ markers[0] - 2 < 16, True)
+ check("the head built runs past head_max_len", len(ids) > 192, True)
+
+
+def test_page_teaches_the_spare_as_the_trigger_not_overflow():
+ """The page must state the cut's condition as the spare budget, and never as an overflow."""
+ _, sentences = _page_prose()
+
+ clauses = [m.group(0) for sent in sentences for m in CONDITIONAL.finditer(sent)]
+ assert clauses, (
+ "the page states no conditional clause at all, so the rules below would pass by reading "
+ "nothing: it has to describe when the budget cuts")
+ bad = [c for c in clauses if OVERFLOW.search(c)]
+ assert not bad, (
+ "the page makes the options overflowing the head the trigger (%s). `build_head` cuts on "
+ "`opt_budget < 16`, which happens while the options still fit the head -- see "
+ "test_the_option_cut_waits_on_spare_tokens_not_on_overflow" % bad)
+ stated = [c for c in clauses if "16" in c]
+ assert stated, (
+ "no condition on the page names the 16 tokens the cut waits for, so the trigger it teaches "
+ "is not the one `build_head` implements")
+
+
+def test_page_does_not_reserve_sixteen_for_the_instruction():
+ """`16` is the spare threshold and the division's reserve; the instruction's floor is 8."""
+ joined, sentences = _page_prose()
+
+ formula = INSTRUCTION_FORMULA.search(joined)
+ assert formula, (
+ "the page no longer shows the instruction clip (`head_ids[: max(8, opt_budget)]`), so its "
+ "reader cannot see what the instruction is actually held to")
+ floor = [sent for sent in sentences if "instruction" in sent.lower() and INSTRUCTION_FLOOR_8.search(sent)]
+ assert floor, "no sentence gives the instruction its floor as a number"
+ wrong = [sent for sent in sentences if RESERVE_16_FOR_INSTRUCTION.search(sent)]
+ assert not wrong, (
+ "the page calls 16 the instruction's reserve or minimum (%s). It is not: the instruction "
+ "keeps `max(8, opt_budget)` tokens, and a 77-option question leaves it with 8 -- see "
+ "test_the_instruction_floor_is_eight_not_the_reserved_sixteen" % wrong)
+ role = [sent for sent in sentences
+ if re.search(r"\b16\b", sent) and re.search(r"threshold|spare|keeps back|reserves? the "
+ r"division|division", sent, re.I)]
+ assert role, "the page never says what the 16 in its own formula is for"
+
+
+def test_page_does_not_put_the_option_block_inside_the_cap():
+ """`head_max_len` is a target the head can pass, and the page has to say so."""
+ joined, sentences = _page_prose()
+
+ inside = [sent for sent in sentences if BLOCK_INSIDE_CAP.search(sent)]
+ assert not inside, (
+ "the page puts the option block inside `head_max_len` (%s). Past roughly head_max_len / 4 "
+ "options every one sits at the 4-token floor and the head grows past the cap -- 319 tokens "
+ "against a 192 cap for the question this page runs -- so the state is sized from the head "
+ "built, not from the cap" % inside)
+ honest = [sent for sent in sentences if CAP_IS_NOT_HARD.search(sent)]
+ assert honest, (
+ "nothing on the page says the head can run past `head_max_len`, which is the consequence "
+ "a high-cardinality reader needs")
+
+
+def test_page_arithmetic_is_build_head_arithmetic():
+ """Every number the page's formulas carry is the number `build_head` carries."""
+ joined, _ = _page_prose()
+ code = _budget_literals()
+
+ per = PER_FORMULA.search(joined)
+ assert per, (
+ "the page no longer states the cut as `per = max(4, (head_max_len - 16) // N)`, so this "
+ "suite cannot check it against build_head")
+ instruction = INSTRUCTION_FORMULA.search(joined)
+ assert instruction, (
+ "the page no longer states the instruction clip as `head_ids[: max(8, opt_budget)]`, so "
+ "this suite cannot check that number either")
+ check("per-option floor on the page", int(per.group(1)), code["per_floor"])
+ check("reserve on the page", int(per.group(2)), code["reserve"])
+ check("instruction floor on the page", int(instruction.group(1)), code["instruction_floor"])
+ assert code["threshold"] == code["reserve"], (
+ "build_head now triggers on a spare of %s but reserves %s in the division: the page states "
+ "one number where the code has two, so its prose has to say which is which"
+ % (code["threshold"], code["reserve"]))
+
+ worked = WORKED_INSTANCE.findall(joined)
+ assert worked, "the page states no worked instance of its own formula"
+ for floor, head, reserve, options, result in worked:
+ want = max(int(floor), (int(head) - int(reserve)) // max(1, int(options)))
+ check("worked instance max(%s, (%s-%s)//%s)" % (floor, head, reserve, options),
+ int(result), want)
+
+
if __name__ == "__main__":
for fn in (test_state_budget_is_sized_from_the_actual_head,
test_dropping_the_minus_one_is_caught,
test_cap_based_figure_understates_a_short_question,
test_cap_based_figure_overstates_a_high_cardinality_question,
+ test_the_option_cut_waits_on_spare_tokens_not_on_overflow,
+ test_the_instruction_floor_is_eight_not_the_reserved_sixteen,
+ test_page_teaches_the_spare_as_the_trigger_not_overflow,
+ test_page_does_not_reserve_sixteen_for_the_instruction,
+ test_page_does_not_put_the_option_block_inside_the_cap,
+ test_page_arithmetic_is_build_head_arithmetic,
test_readme_quotes_the_measured_figures):
+
try:
fn()
except AssertionError as e: