Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(predict_long): the budget check must not be a hook, and must cover both agents
Adeshina Lasisiee3f9d43f6f9
diff --git a/laya/agent.py b/laya/agent.pyindex 69f8d34b..f336e44a 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -267,7 +267,8 @@ def _start_evidence():inference, `states` is a snapshot of the states that reached it (`None` if the probe never ran,which means `predict_batch` was replaced and no hook chain was dispatched)."""- evidence = {"answered": False, "states": None, "question_types": None}+ evidence = {"answered": False, "states": None, "question_types": None,+ "max_len": None, "head_max_len": None, "questions": None}def probe(ctx):evidence["answered"] = ctx.results is not None@@ -279,46 +280,65 @@ def _start_evidence():qid: qdef.get("type") if isinstance(qdef, dict) else Nonefor qid, qdef in ctx.questions.items()}+ # Also what the budget and the questions ACTUALLY are once the chain has run, for+ # `_check_scan_budget`. Recorded, never judged here: a hook that raises is governed by the+ # caller's `hooks_raise`/`hooks_timeout`, so a correctness check that raises from inside the+ # chain can be switched off by a policy meant for third-party telemetry. Measured: with+ # `hooks_raise=False` -- what `docs/hooks/tracing.md` recommends -- a refusal became a+ # RuntimeWarning blaming `_StartAdapter`, and the scan proceeded.+ evidence["max_len"] = ctx.max_len+ evidence["head_max_len"] = ctx.head_max_len+ evidence["questions"] = dict(ctx.questions) if ctx.questions else {}return probe, evidence-def _window_budget_guard(tok, internal, sized, cfg_max_len, cfg_head_max_len):- """A start hook that refuses a scan whose windows no longer fit the budget in force.-- `predict_long` sizes its windows from the agent's config, but a start hook may set- `ctx.max_len` / `ctx.head_max_len`, and those are what `build_sequence` uses. When the hook- widens the head more than it widens `max_len` -- which `widen_for_high_cardinality` in- `docs/hooks/patterns.md` does, and it is documented for exactly the high-cardinality questions- that make windowing necessary -- the real room SHRINKS, every window is re-truncated on the way- in, and once the stride exceeds the real room consecutive windows stop touching. Measured on the- English checkpoint with that hook at 50 options: windows sized 303, room actually 253; at 100- options the reviewer measured 43.4% of a document reaching no model, against the 37.6% this- change exists to remove. Silently worse than not windowing at all.-- Sizing after the hook chain would mean dispatching it before the windows exist, which changes- when hooks run and what they see. Refusing is the honest alternative: the scan cannot honour a- budget it was not sized for, and a caller who needs both can pass `window=`/`stride=` explicitly- or drive `predict_batch` itself. This runs in the start chain, so it aborts before any forward- pass rather than after one.+def _check_scan_budget(agent, evidence, sized, cfg_max_len, cfg_head_max_len):+ """Raise when the budget or questions in force leave less room than the scan was sized for.++ `predict_long` sizes its windows from the agent's config, before any hook has run. A start hook+ may then set `ctx.max_len`/`ctx.head_max_len`, or rewrite `ctx.questions` -- both documented+ powers -- and `build_sequence` uses whatever it finds. When the head widens faster than+ `max_len`, or the questions get more options, the real room SHRINKS: every window is+ re-truncated on the way in, and once the stride exceeds the real room consecutive windows stop+ touching. Measured on the English checkpoint with `widen_for_high_cardinality` from+ `docs/hooks/patterns.md` at 50 options, windows sized 303 against a room of 253; at 100 options+ a reviewer measured 43.4% of a document reaching no model, against the 37.6% this change exists+ to remove. Silently worse than not windowing at all.++ Called by `predict_long` AFTER the chain has run, not raised from inside it. A hook that raises+ is subject to the caller's `hooks_raise` and `hooks_timeout`, so the first version of this check+ could be switched off by `hooks_raise=False` -- which `docs/hooks/tracing.md` recommends -- and+ could blow a `hooks_timeout` the caller set for their own hooks. The cost is that the forward+ pass has already happened when this fires: a misconfigured scan is refused rather than answered+ wrongly, but it is not refused for free. Sizing the windows after the chain instead would change+ when hooks run and what they see, which is a larger change than this one."""- def guard(ctx):- eff_max_len = ctx.max_len if ctx.max_len is not None else cfg_max_len- eff_head = ctx.head_max_len if ctx.head_max_len is not None else cfg_head_max_len- if eff_max_len == cfg_max_len and eff_head == cfg_head_max_len:- return- room = min(state_room(tok, q, eff_max_len, eff_head) for q in internal)- if room >= sized:- return # the budget moved, but not in a direction that truncates- raise ValueError(- "predict_long: a start hook set the token budget to max_len=%d head_max_len=%d, which "- "leaves %d state tokens per window, but the scan was sized for %d from the agent's "- "config (max_len=%d head_max_len=%d). Every window would be re-truncated and parts of "- "the document would reach no model. Pass window=/stride= explicitly, or call "- "predict_batch directly, if you need both."- % (eff_max_len, eff_head, room, sized, cfg_max_len, cfg_head_max_len))-- return guard+ if evidence.get("answered"):+ return # a hook answered the document; no window was scored+ questions = evidence.get("questions")+ if not questions:+ return # nothing to fit, as `window_budget` also concludes+ eff_max_len = evidence.get("max_len")+ eff_head = evidence.get("head_max_len")+ eff_max_len = cfg_max_len if eff_max_len is None else eff_max_len+ eff_head = cfg_head_max_len if eff_head is None else eff_head+ try:+ internal = [agent._to_internal(q) for q in questions.values()]+ except Exception:+ return # malformed questions are the validator's to report+ if not internal:+ return+ room = min(state_room(agent.tok, q, eff_max_len, eff_head) for q in internal)+ if room >= sized:+ return+ raise ValueError(+ "predict_long: after the start hooks ran, the questions and token budget (max_len=%d "+ "head_max_len=%d) leave %d state tokens per window, but the scan was sized for %d from the "+ "agent's config (max_len=%d head_max_len=%d). Every window would be re-truncated and parts "+ "of the document would reach no model. Pass window=/stride= explicitly, or call "+ "predict_batch directly, if you need a start hook to change either."+ % (eff_max_len, eff_head, room, sized, cfg_max_len, cfg_head_max_len))def _with_start_probe(hook_kwargs, probe):@@ -1462,6 +1482,11 @@ class Agent(HookRegistry):# Why 0 for a hook answer here: the state did fit one window, but no window was scored,# which is the same fact the multi-window path reports as 0. Reading 1 would make a# cached answer and a served answer agree on how much of the input the model saw.+ # The same budget check as the multi-window path. Without it a document short enough to+ # fit one window was silently truncated by a re-budgeting hook and still reported+ # `windows: 1`, i.e. "the model read all of it" -- measured, 138 of 240 state tokens+ # never reached the model, while a longer document on the identical input hard-failed.+ _check_scan_budget(self, evidence, budget, max_len, head_max_len)single["usage"] = {**(single.get("usage") or {}), "windows": 0 if evidence["answered"] else 1}return single@@ -1478,8 +1503,6 @@ class Agent(HookRegistry):i += stepprobe, evidence = _start_evidence()- guard = _window_budget_guard(self.tok, [internal[qid] for qid in ids], budget,- max_len, head_max_len)# `list(windows)`, not `windows`: `ctx.states` is the list the hook receives, so a hook that# mutates it in place (`append`, `sort`) would otherwise also grow the split this call# attributes answers to, and the counts would agree while `starts` no longer lined up.@@ -1488,7 +1511,8 @@ class Agent(HookRegistry):cap = window_batch_cap(len(windows), budget, max(64, max_len - head_max_len - 8),batch_size)results = self.predict_batch(list(windows), questions, batch_size=cap, lang=lang,- **_with_start_probe(hook_kwargs, [guard, probe]))+ **_with_start_probe(hook_kwargs, probe))+ _check_scan_budget(self, evidence, budget, max_len, head_max_len)if evidence["answered"]:# The hook replaced the call before any window was scored. Aggregating over its payloaddiff --git a/laya/onnx_agent.py b/laya/onnx_agent.pyindex 50ccbd67..23ec2ccd 100644--- a/laya/onnx_agent.py+++ b/laya/onnx_agent.py@@ -440,7 +440,7 @@ class ONNXAgent(HookRegistry):in `Agent.predict_long`: `truncated` is a window count and `truncated_questions` is thelast window's list, so `truncated` can be above 0 while the list is empty."""- from .agent import _start_evidence, _with_start_probe+ from .agent import _check_scan_budget, _start_evidence, _with_start_probefrom .hooks import aggregate_usagehook_kwargs = {"hooks": hooks, "on_predict_start": on_predict_start,@@ -486,6 +486,11 @@ class ONNXAgent(HookRegistry):probe, evidence = _start_evidence()single = dict(self.system_one(state, questions, lang=lang,**_with_start_probe(hook_kwargs, probe)))+ # The same budget check as the multi-window path. Without it a document short enough to+ # fit one window was silently truncated by a re-budgeting hook and still reported+ # `windows: 1`, i.e. "the model read all of it" -- measured, 138 of 240 state tokens+ # never reached the model, while a longer document on the identical input hard-failed.+ _check_scan_budget(self, evidence, budget, max_len, head_max_len)single["usage"] = {**(single.get("usage") or {}), "windows": 0 if evidence["answered"] else 1}return single@@ -509,6 +514,12 @@ class ONNXAgent(HookRegistry):batch_size)results = self.predict_batch(list(windows), questions, batch_size=cap, lang=lang,**_with_start_probe(hook_kwargs, probe))+ # Same check as the torch agent, for the same reason and on the same contract: a start hook+ # re-budgets an ONNX scan exactly as it re-budgets a torch one (`predict_batch` applies+ # `ctx.max_len`/`ctx.head_max_len` identically), so leaving it off here meant the bug was+ # fully live on this path while the other agent refused the identical input -- measured,+ # 34.6% of a document reaching no model.+ _check_scan_budget(self, evidence, budget, max_len, head_max_len)if evidence["answered"]:# A hook answered the document before any window was scored: pass that answerdiff --git a/tests/test_predict_long.py b/tests/test_predict_long.pyindex 219335c5..6ea2a7ff 100644--- a/tests/test_predict_long.py+++ b/tests/test_predict_long.py@@ -17,7 +17,8 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))import numpy as np # noqa: E402-from laya.agent import Agent, _window_budget_guard+import laya.agent as agent_mod+from laya.agent import Agent, _check_scan_budget, _start_evidencefrom laya import commonfrom laya.onnx_agent import ONNXAgent # noqa: E402@@ -847,79 +848,133 @@ check_true("batch cap/the torch scan passes the cap to predict_batch","batch_size=cap" in _torch_src, "")check_true("batch cap/the ONNX scan passes the cap to predict_batch","window_batch_cap(" in _onnx_src and "batch_size=cap" in _onnx_src, "")-# Constructing it is not installing it: removing only the hand-off left the construction in place-# and a "is it mentioned?" check green.-check_true("hook budget/the torch scan installs the budget guard",- "_window_budget_guard(" in _torch_src and "[guard, probe]" in _torch_src, "")# Compared on the statements, not on any mention: the comment above the call names `_to_internal`.check_true("questions/the ONNX scan validates before _to_internal",_onnx_src.index("_Agent._check_question(qid")< _onnx_src.index("internal = {qid: self._to_internal"), "")-# A start hook may set ctx.max_len / ctx.head_max_len, and those are what `build_sequence` uses ---# but the scan was sized from the agent's config. `widen_for_high_cardinality`, which-# docs/hooks/patterns.md ships for exactly these questions, widens the head more than max_len, so-# the real room SHRINKS: measured on the English checkpoint at 50 options, windows sized 303 against-# a real room of 253, and at 100 options 43.4% of the document reached no model -- worse than the-# 37.6% this change removes. Refusing is the honest answer; silently scanning is not.-def _widen_head_only(ctx):- ctx.head_max_len = 60 # cfg is max_len=100 head_max_len=20, so the room shrinks---def _widen_both(ctx):- ctx.max_len, ctx.head_max_len = 400, 60 # widened together: the room grows+# A start hook may set ctx.max_len / ctx.head_max_len, or rewrite ctx.questions -- both documented+# powers -- and `build_sequence` uses whatever it finds, while the scan was sized from the config+# before any hook ran. `widen_for_high_cardinality`, which docs/hooks/patterns.md ships for exactly+# these questions, widens the head faster than max_len, so the real room SHRINKS: 303 sized against+# 253 real at 50 options on the English checkpoint, and 43.4% of a document unread at 100.+#+# The first version of this was a start hook that RAISED, and three ways of defeating it all left+# the suite green: `hooks_raise=False` -- what docs/hooks/tracing.md recommends -- downgraded the+# refusal to a RuntimeWarning and the scan proceeded; a hook that rewrote ctx.questions instead of+# the budget was invisible; and `sized=0` at the call site made it unable to fire while the+# source-text assertions still matched. It is now a plain function `predict_long` calls itself, so+# no hook policy governs it, and both halves are checked below: what it decides, and that it is+# actually called with the budget the scan used.+_CFG = (100, 20) # make_agent's config; budget = max(64, 100-20-8) = 72-_guard = _window_budget_guard(TOK, [Agent._to_internal(q_many(12))],- sized=72, cfg_max_len=100, cfg_head_max_len=20)+def _evidence(max_len=None, head_max_len=None, questions=None, answered=False):+ return {"answered": answered, "states": None, "max_len": max_len,+ "head_max_len": head_max_len,+ "questions": {"a": q_many(2)} if questions is None else questions}-class _Ctx:- def __init__(self, max_len=None, head_max_len=None):- self.max_len, self.head_max_len = max_len, head_max_len+_probe_agent = make_agent(lambda states, qs: [])+def _scan_raw(questions=None, **kw):+ a = make_agent(lambda states, qs: [])+ return a.predict_long(LONG, questions or {"a": q_many(2)}, **kw)-def _zero_room_message():- try:- common.window_budget(TOK, [Agent._to_internal(q_many(24))], 100, 20,- window=None, stride=None)- except ValueError as exc:- return str(exc)- return ""-def _guard_message():- """The text of the refusal, so the message itself is asserted rather than just the type."""+def _budget_message():try:- _guard(_Ctx(100, 60))+ _check_scan_budget(_probe_agent, _evidence(head_max_len=60), 72, *_CFG)except ValueError as exc:return str(exc)return ""+check_raises("hook budget/a hook that shrinks the room is refused", ValueError,+ lambda: _check_scan_budget(_probe_agent, _evidence(head_max_len=60), 72, *_CFG))+check_raises("hook budget/rewriting ctx.questions is caught too", ValueError,+ lambda: _check_scan_budget(_probe_agent, _evidence(questions={"a": q_many(16)}),+ 72, *_CFG))check_true("hook budget/an unchanged budget is allowed",- _guard(_Ctx()) is None, "no hook")+ _check_scan_budget(_probe_agent, _evidence(), 72, *_CFG) is None, "no hook")check_true("hook budget/a budget widened together is allowed",- _guard(_Ctx(400, 60)) is None, "room grows")-check_raises("hook budget/a hook that shrinks the room is refused", ValueError,- lambda: _guard(_Ctx(100, 60)))-# The limitation this replaces: one oversized question aborts a whole multi-question call, because-# the budget takes min(rooms). Untested before -- `if room <= 0:` -> `if room < 0:` can never fire,-# since `state_room` clamps at 0, and every suite stayed green.-check("room/24 options leave exactly zero state tokens",- common.state_room(TOK, Agent._to_internal(q_many(24)), 100, 20), 0)-check_raises("room/questions that fill the sequence are refused, not silently truncated", ValueError,- lambda: common.window_budget(TOK, [Agent._to_internal(q_many(24))], 100, 20,- window=None, stride=None))-# `room <= 0` must be the boundary: `room < 0` can never fire, since `state_room` clamps at 0, so-# the refusal would be dead code and a zero-room scan would proceed. Asserted on the message, which-# only the `<= 0` branch produces.-check_true("room/the refusal is the zero-room branch, not an incidental error",- "no window can carry any of it" in _zero_room_message(), _zero_room_message())+ _check_scan_budget(_probe_agent, _evidence(400, 60), 72, *_CFG) is None, "room grows")+check_true("hook budget/a hook that answered the document is not refused",+ _check_scan_budget(_probe_agent, _evidence(head_max_len=60, answered=True),+ 72, *_CFG) is None, "cache hit")+check_true("hook budget/no questions is not a crash",+ _check_scan_budget(_probe_agent, _evidence(head_max_len=60, questions={}),+ 72, *_CFG) is None, "empty questions")check_true("hook budget/the refusal names both budgets and both rooms",- all(t in _guard_message() for t in ("max_len=100", "head_max_len=60", "sized for 72")),- _guard_message())+ all(t in _budget_message() for t in ("max_len=100", "head_max_len=60", "sized for 72")),+ _budget_message())++# ... and that predict_long actually calls it, with the budget it sized the windows to. This is what+# `sized=0` walked past: the symbol was present and the call site spelled right, but the value made+# the check inert.+_seen_budget = []+_real_check = agent_mod._check_scan_budget+try:+ agent_mod._check_scan_budget = lambda ag, ev, sized, ml, hm: _seen_budget.append((sized, ml, hm))+ _wired = make_agent(lambda states, qs: [+ {"model": "m", "answers": {"a": {"choice": "x", "answer_confidence": 0.5}},+ "usage": {"input_tokens": 1}} for _ in states])+ _wired.predict_long(LONG, {"a": q_many(2)})+finally:+ agent_mod._check_scan_budget = _real_check+check("hook budget/the torch scan checks its budget exactly once", len(_seen_budget), 1)+# The room is the SMALLEST any question leaves, which is why `window_budget` takes a min. No test+# put two questions of different cardinality through the checker, so `min` -> `max` was green.+# At the unchanged budget these two leave rooms of 76 and 24 against a scan sized for 72, so `min`+# refuses and `max` does not -- which is the whole reason `window_budget` takes a min. Collapsing+# them under a widened head (both 36) makes the mutant survive, which is how it slipped through.+check_raises("hook budget/the smallest room wins across questions", ValueError,+ lambda: _check_scan_budget(+ _probe_agent, _evidence(questions={"a": q_many(2), "b": q_many(16)}), 72, *_CFG))++# What the probe RECORDS is half the check: synthesising an evidence dict in the tests above leaves+# the recording itself unpinned, and dropping either field made the checker silently inert.+_rec_probe, _rec = _start_evidence()+++class _RecCtx:+ results = None+ states = ["s"]+ max_len = 400+ head_max_len = 60+ questions = {"a": q_many(2)}+++_rec_probe(_RecCtx())+check("probe/records the budget in force", (_rec["max_len"], _rec["head_max_len"]), (400, 60))+check("probe/records the questions in force", list(_rec["questions"]), ["a"])+check_true("probe/copies the questions rather than aliasing them",+ _rec["questions"] is not _RecCtx.questions, "")++# Validation before `_to_internal` on the torch agent too. The ONNX side is pinned by a source-order+# check; deleting the torch loop was green.+check_raises("questions/the torch scan validates before _to_internal", ValueError,+ lambda: _scan_raw(questions={"a": {"type": "choice", "instructions": "?",+ "criteria": None}}))++# Both agents, not one. The ONNX scan had no check at all while the torch one refused the identical+# input -- measured, 34.6% of a document reaching no model -- and the source comment two screens up+# says the contract binds both agents.+_onnx_src = inspect.getsource(ONNXAgent.predict_long)+check_true("hook budget/the ONNX scan checks its budget too",+ "_check_scan_budget(self, evidence, budget, max_len, head_max_len)" in _onnx_src, "")+check("hook budget/both agents check it on the single-window path too",+ (inspect.getsource(Agent.predict_long).count("_check_scan_budget("),+ _onnx_src.count("_check_scan_budget(")), (2, 2))++check("hook budget/it is handed the window size and the config it sized from",+ _seen_budget[0] if _seen_budget else None,+ (window_budget(TOK, [Agent._to_internal(q_many(2))], 100, 20,+ window=None, stride=None)[0], 100, 20))++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL: