Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
test(state-budget): hold the high-cardinality page to build_head's thresholds and floors
Aashish7474bc3c9215
diff --git a/tests/test_state_budget.py b/tests/test_state_budget.pyindex 65f0f198..d35d4ed3 100644--- a/tests/test_state_budget.py+++ b/tests/test_state_budget.py@@ -16,14 +16,25 @@ from the head it actually built.This suite pins the relationship, and — when the checkpoints are present — reads the figuresback out of `README.md` and checks them against a live measurement, so the renderer, thedocumentation and this file cannot drift apart silently.++`examples/33_high_cardinality_choice.py` is the page that teaches this budget, and it taught three+things the code does not do: that the cut waits for the options to *overflow* `head_max_len` (it+fires on the spare budget, while they still fit), that 16 is a minimum *reserved for the instruction*+(the instruction is held to 8), and that the option block *lives in* `head_max_len` (past ~4 tokens+per option the head grows past the cap and takes the state's room with it). The last sections of+this file pin the page to `build_head` — behaviourally, and by reading the budget's literals out of+the function instead of typing them in here."""+import ast+import inspectimport osimport reimport sys+import textwrapsys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))-from laya.common import build_sequence # noqa: E402+from laya.common import build_head, build_sequence, render_options # noqa: E402PASS, FAIL, SKIPPED = [], [], []@@ -214,12 +225,253 @@ def test_readme_quotes_the_measured_figures():skip("no checkpoints under %s" % _models_dir())+# ---------------------------------------------------- example 33: the page against the budget it teaches++EXAMPLE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),+ "examples", "33_high_cardinality_choice.py")++# The two formulas, in the symbolic form the page has to teach them in.+PER_FORMULA = re.compile(r"per\s*=\s*max\(\s*(\d+)\s*,\s*\(\s*head_max_len\s*-\s*(\d+)\s*\)"+ r"\s*//\s*N\s*\)")+INSTRUCTION_FORMULA = re.compile(r"head_ids\[:\s*max\(\s*(\d+)\s*,\s*opt_budget\s*\)\]")+# `max(4, (192-16)//77) = 4` -- a worked instance, whose arithmetic has to come out right.+WORKED_INSTANCE = re.compile(r"max\(\s*(\d+)\s*,\s*\((\d+)\s*-\s*(\d+)\)\s*//\s*(\d+)\s*\)"+ r"\s*=\s*(\d+)")+CONDITIONAL = re.compile(r"\b(?:when|once|if|unless)\b[^,;.]{0,80}", re.I)+OVERFLOW = re.compile(r"overflow", re.I)+# 16 taught as the instruction's reserve: the number that is really the instruction's floor is 8.+RESERVE_16_FOR_INSTRUCTION = re.compile(+ r"\b16\b[^.]{0,60}\b(?:reserv\w*|minimum)\b[^.]{0,60}instruction"+ r"|instruction[^.]{0,60}\b(?:reserv\w*|minimum)\b[^.]{0,60}\b16\b", re.I)+BLOCK_INSIDE_CAP = re.compile(+ r"\b(?:lives|live|fits|fit|stays|stay|kept)\b[^.]{0,30}\b(?:in|within|under|inside)\b[^.]{0,30}"+ r"head_max_len", re.I)+CAP_IS_NOT_HARD = re.compile(r"not a hard cap|no hard cap|past the cap|past head_max_len"+ r"|longer than head_max_len|head longer than|grows past", re.I)+INSTRUCTION_FLOOR_8 = re.compile(r"max\(\s*8\s*,|8-token|\b8\b[^.]{0,40}instruction"+ r"|instruction[^.]{0,40}\b8\b", re.I)+++def _page_prose():+ """The example's string constants, whitespace-normalised, as (joined, sentences).++ Sentences are split inside each constant, never over a join of them: two adjacent constants+ can read as one sentence, which would let a true sentence vouch for a false neighbour.+ """+ if not os.path.exists(EXAMPLE):+ raise AssertionError("examples/33_high_cardinality_choice.py is not where this suite looks "+ "for it (%s)" % EXAMPLE)+ with open(EXAMPLE, encoding="utf-8") as fh:+ tree = ast.parse(fh.read())+ constants = [re.sub(r"\s+", " ", node.value) for node in ast.walk(tree)+ if isinstance(node, ast.Constant) and isinstance(node.value, str)]+ sentences = [sent.strip() for text in constants+ for sent in re.split(r"(?<=[.!?])\s+", text) if sent.strip()]+ return " ".join(constants), sentences+++def _option_question(lengths, ins="Which one?"):+ """A choice question whose options cost exactly `lengths` tokens under `_FakeTok`.++ One id per word, and an option renders as `label: <criterion>`, so the criterion carries+ `length - 2` words: the label and its colon are the other two.+ """+ return {"t": "choice", "ins": ins,+ "crit": {"o%d" % i: " ".join(["w"] * (length - 2)) for i, length in enumerate(lengths)}}+++def _full_option_len(tok, q):+ """Tokens each option costs before the budget cuts it, `[MASK]` included."""+ return [1 + len(tok(" " + opt)["input_ids"]) for opt in render_options(q)]+++def _budget_literals():+ """(threshold, reserve, per-floor, instruction-floor) exactly as `build_head` writes them.++ Read out of the function rather than typed in here, so a page holding a stale number fails+ instead of agreeing with itself.+ """+ tree = ast.parse(textwrap.dedent(inspect.getsource(build_head)))+ found = {}+ for node in ast.walk(tree):+ if isinstance(node, ast.Compare) and len(node.ops) == 1 and isinstance(node.ops[0], ast.Lt):+ left, right = node.left, node.comparators[0]+ if isinstance(left, ast.Name) and left.id == "opt_budget" and isinstance(right, ast.Constant):+ found["threshold"] = right.value+ if isinstance(node, ast.Call) and getattr(node.func, "id", "") == "max" and len(node.args) == 2:+ first, second = node.args+ if not isinstance(first, ast.Constant):+ continue+ if isinstance(second, ast.BinOp) and isinstance(second.op, ast.FloorDiv) \+ and isinstance(second.left, ast.BinOp) and isinstance(second.left.op, ast.Sub) \+ and getattr(second.left.left, "id", "") == "head_max_len" \+ and isinstance(second.left.right, ast.Constant):+ found["per_floor"] = first.value+ found["reserve"] = second.left.right.value+ if isinstance(second, ast.Name) and second.id == "opt_budget":+ found["instruction_floor"] = first.value+ missing = {"threshold", "reserve", "per_floor", "instruction_floor"} - set(found)+ assert not missing, (+ "build_head no longer writes the budget the way this suite reads it: %s missing"+ % sorted(missing))+ return found+++def test_the_option_cut_waits_on_spare_tokens_not_on_overflow():+ """The cut fires on the leftover head budget, and fires while the options still fit.++ This is the ground truth the page has to match: `head_max_len - sum(options) < 16`, not the+ options overflowing `head_max_len`. A question whose options fill the head exactly -- 192 of+ 192, nothing dropped -- is already cut.+ """+ tok = _FakeTok()+ head_max_len = 192++ fills_it = _option_question([48, 48, 48, 48])+ full = _full_option_len(tok, fills_it)+ check("fixture: four options of 48 fill the head without overflowing it",+ (sum(full), sum(full) > head_max_len, head_max_len - sum(full)), (192, False, 0))+ _, _, stats = build_head(tok, fills_it, head_max_len)+ assert stats["tokens_per_option"] is not None, (+ "a question that leaves 0 spare tokens must already be cut to "+ "max(4, (head_max_len - 16) // N); it kept its full lengths, so the cut is not where "+ "this suite says it is")+ check("cut size for 4 options at head_max_len=192", stats["tokens_per_option"], 44)++ spare_16 = _option_question([44, 44, 44, 44])+ check("fixture: 16 spare tokens", head_max_len - sum(_full_option_len(tok, spare_16)), 16)+ _, _, stats_16 = build_head(tok, spare_16, head_max_len)+ check("16 spare tokens is not yet the cut", stats_16["tokens_per_option"], None)++ spare_15 = _option_question([45, 44, 44, 44])+ check("fixture: 15 spare tokens", head_max_len - sum(_full_option_len(tok, spare_15)), 15)+ _, _, stats_15 = build_head(tok, spare_15, head_max_len)+ assert stats_15["tokens_per_option"] is not None, (+ "15 spare tokens is under the 16 the cut waits for, so this must be cut")+++def test_the_instruction_floor_is_eight_not_the_reserved_sixteen():+ """Past the floor the instruction keeps 8 tokens -- 16 is not reserved for it.++ 77 options at the 4-token floor leave the head short of itself, and the instruction is clipped+ to `max(8, opt_budget)` with a negative `opt_budget`: the retained prefix is 8, from a question+ whose instruction is far longer.+ """+ tok = _FakeTok()+ ins = ("Which of these many clearly distinct banking intents best matches the customer "+ "message right now, please answer carefully")+ q = _many_options(77)+ q["ins"] = ins+ full_instruction = len(tok("%s question: %s" % (q["t"], ins))["input_ids"])+ assert full_instruction > 16, (+ "fixture needs an instruction long enough that a 16-token reserve would show up")++ ids, markers, stats = build_head(tok, q, 192)+ check("77 options are cut to the per-option floor", stats["tokens_per_option"], 4)+ check("instruction tokens retained", markers[0] - 2, 8)+ check("the retained prefix is below the 16 the page must not call its reserve",+ markers[0] - 2 < 16, True)+ check("the head built runs past head_max_len", len(ids) > 192, True)+++def test_page_teaches_the_spare_as_the_trigger_not_overflow():+ """The page must state the cut's condition as the spare budget, and never as an overflow."""+ _, sentences = _page_prose()++ clauses = [m.group(0) for sent in sentences for m in CONDITIONAL.finditer(sent)]+ assert clauses, (+ "the page states no conditional clause at all, so the rules below would pass by reading "+ "nothing: it has to describe when the budget cuts")+ bad = [c for c in clauses if OVERFLOW.search(c)]+ assert not bad, (+ "the page makes the options overflowing the head the trigger (%s). `build_head` cuts on "+ "`opt_budget < 16`, which happens while the options still fit the head -- see "+ "test_the_option_cut_waits_on_spare_tokens_not_on_overflow" % bad)+ stated = [c for c in clauses if "16" in c]+ assert stated, (+ "no condition on the page names the 16 tokens the cut waits for, so the trigger it teaches "+ "is not the one `build_head` implements")+++def test_page_does_not_reserve_sixteen_for_the_instruction():+ """`16` is the spare threshold and the division's reserve; the instruction's floor is 8."""+ joined, sentences = _page_prose()++ formula = INSTRUCTION_FORMULA.search(joined)+ assert formula, (+ "the page no longer shows the instruction clip (`head_ids[: max(8, opt_budget)]`), so its "+ "reader cannot see what the instruction is actually held to")+ floor = [sent for sent in sentences if "instruction" in sent.lower() and INSTRUCTION_FLOOR_8.search(sent)]+ assert floor, "no sentence gives the instruction its floor as a number"+ wrong = [sent for sent in sentences if RESERVE_16_FOR_INSTRUCTION.search(sent)]+ assert not wrong, (+ "the page calls 16 the instruction's reserve or minimum (%s). It is not: the instruction "+ "keeps `max(8, opt_budget)` tokens, and a 77-option question leaves it with 8 -- see "+ "test_the_instruction_floor_is_eight_not_the_reserved_sixteen" % wrong)+ role = [sent for sent in sentences+ if re.search(r"\b16\b", sent) and re.search(r"threshold|spare|keeps back|reserves? the "+ r"division|division", sent, re.I)]+ assert role, "the page never says what the 16 in its own formula is for"+++def test_page_does_not_put_the_option_block_inside_the_cap():+ """`head_max_len` is a target the head can pass, and the page has to say so."""+ joined, sentences = _page_prose()++ inside = [sent for sent in sentences if BLOCK_INSIDE_CAP.search(sent)]+ assert not inside, (+ "the page puts the option block inside `head_max_len` (%s). Past roughly head_max_len / 4 "+ "options every one sits at the 4-token floor and the head grows past the cap -- 319 tokens "+ "against a 192 cap for the question this page runs -- so the state is sized from the head "+ "built, not from the cap" % inside)+ honest = [sent for sent in sentences if CAP_IS_NOT_HARD.search(sent)]+ assert honest, (+ "nothing on the page says the head can run past `head_max_len`, which is the consequence "+ "a high-cardinality reader needs")+++def test_page_arithmetic_is_build_head_arithmetic():+ """Every number the page's formulas carry is the number `build_head` carries."""+ joined, _ = _page_prose()+ code = _budget_literals()++ per = PER_FORMULA.search(joined)+ assert per, (+ "the page no longer states the cut as `per = max(4, (head_max_len - 16) // N)`, so this "+ "suite cannot check it against build_head")+ instruction = INSTRUCTION_FORMULA.search(joined)+ assert instruction, (+ "the page no longer states the instruction clip as `head_ids[: max(8, opt_budget)]`, so "+ "this suite cannot check that number either")+ check("per-option floor on the page", int(per.group(1)), code["per_floor"])+ check("reserve on the page", int(per.group(2)), code["reserve"])+ check("instruction floor on the page", int(instruction.group(1)), code["instruction_floor"])+ assert code["threshold"] == code["reserve"], (+ "build_head now triggers on a spare of %s but reserves %s in the division: the page states "+ "one number where the code has two, so its prose has to say which is which"+ % (code["threshold"], code["reserve"]))++ worked = WORKED_INSTANCE.findall(joined)+ assert worked, "the page states no worked instance of its own formula"+ for floor, head, reserve, options, result in worked:+ want = max(int(floor), (int(head) - int(reserve)) // max(1, int(options)))+ check("worked instance max(%s, (%s-%s)//%s)" % (floor, head, reserve, options),+ int(result), want)++if __name__ == "__main__":for fn in (test_state_budget_is_sized_from_the_actual_head,test_dropping_the_minus_one_is_caught,test_cap_based_figure_understates_a_short_question,test_cap_based_figure_overstates_a_high_cardinality_question,+ test_the_option_cut_waits_on_spare_tokens_not_on_overflow,+ test_the_instruction_floor_is_eight_not_the_reserved_sixteen,+ test_page_teaches_the_spare_as_the_trigger_not_overflow,+ test_page_does_not_reserve_sixteen_for_the_instruction,+ test_page_does_not_put_the_option_block_inside_the_cap,+ test_page_arithmetic_is_build_head_arithmetic,test_readme_quotes_the_measured_figures):+try:fn()except AssertionError as e: