Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(predict_long): refuse a scan a start hook has re-budgeted, and restore ONNX validation
Adeshina Lasisi20cb8b1de53d
diff --git a/laya/agent.py b/laya/agent.pyindex f8229fbb..69f8d34b 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -42,6 +42,7 @@ from .common import (temp_bucket,unpermute_probs,window_batch_cap,+ state_room,window_budget,)from .confidence import check_min_confidence, flag_low_confidence@@ -282,10 +283,52 @@ def _start_evidence():return probe, evidence+def _window_budget_guard(tok, internal, sized, cfg_max_len, cfg_head_max_len):+ """A start hook that refuses a scan whose windows no longer fit the budget in force.++ `predict_long` sizes its windows from the agent's config, but a start hook may set+ `ctx.max_len` / `ctx.head_max_len`, and those are what `build_sequence` uses. When the hook+ widens the head more than it widens `max_len` -- which `widen_for_high_cardinality` in+ `docs/hooks/patterns.md` does, and it is documented for exactly the high-cardinality questions+ that make windowing necessary -- the real room SHRINKS, every window is re-truncated on the way+ in, and once the stride exceeds the real room consecutive windows stop touching. Measured on the+ English checkpoint with that hook at 50 options: windows sized 303, room actually 253; at 100+ options the reviewer measured 43.4% of a document reaching no model, against the 37.6% this+ change exists to remove. Silently worse than not windowing at all.++ Sizing after the hook chain would mean dispatching it before the windows exist, which changes+ when hooks run and what they see. Refusing is the honest alternative: the scan cannot honour a+ budget it was not sized for, and a caller who needs both can pass `window=`/`stride=` explicitly+ or drive `predict_batch` itself. This runs in the start chain, so it aborts before any forward+ pass rather than after one.+ """+ def guard(ctx):+ eff_max_len = ctx.max_len if ctx.max_len is not None else cfg_max_len+ eff_head = ctx.head_max_len if ctx.head_max_len is not None else cfg_head_max_len+ if eff_max_len == cfg_max_len and eff_head == cfg_head_max_len:+ return+ room = min(state_room(tok, q, eff_max_len, eff_head) for q in internal)+ if room >= sized:+ return # the budget moved, but not in a direction that truncates+ raise ValueError(+ "predict_long: a start hook set the token budget to max_len=%d head_max_len=%d, which "+ "leaves %d state tokens per window, but the scan was sized for %d from the agent's "+ "config (max_len=%d head_max_len=%d). Every window would be re-truncated and parts of "+ "the document would reach no model. Pass window=/stride= explicitly, or call "+ "predict_batch directly, if you need both."+ % (eff_max_len, eff_head, room, sized, cfg_max_len, cfg_head_max_len))++ return guard++def _with_start_probe(hook_kwargs, probe):- """`hook_kwargs` with `probe` appended after the caller's own start hooks."""+ """`hook_kwargs` with `probe` appended after the caller's own start hooks.++ `probe` may be one hook or a list of them, appended in order.+ """kwargs = dict(hook_kwargs)- kwargs["on_predict_start"] = list(_as_sequence(hook_kwargs.get("on_predict_start"))) + [probe]+ extra = list(probe) if isinstance(probe, (list, tuple)) else [probe]+ kwargs["on_predict_start"] = list(_as_sequence(hook_kwargs.get("on_predict_start"))) + extrareturn kwargsdef _option_logits(logits, items, offset):"""Raw per-option logits, the rows `_decode_answers` divides by temperature.@@ -1435,6 +1478,8 @@ class Agent(HookRegistry):i += stepprobe, evidence = _start_evidence()+ guard = _window_budget_guard(self.tok, [internal[qid] for qid in ids], budget,+ max_len, head_max_len)# `list(windows)`, not `windows`: `ctx.states` is the list the hook receives, so a hook that# mutates it in place (`append`, `sort`) would otherwise also grow the split this call# attributes answers to, and the counts would agree while `starts` no longer lined up.@@ -1443,7 +1488,7 @@ class Agent(HookRegistry):cap = window_batch_cap(len(windows), budget, max(64, max_len - head_max_len - 8),batch_size)results = self.predict_batch(list(windows), questions, batch_size=cap, lang=lang,- **_with_start_probe(hook_kwargs, probe))+ **_with_start_probe(hook_kwargs, [guard, probe]))if evidence["answered"]:# The hook replaced the call before any window was scored. Aggregating over its payloaddiff --git a/laya/onnx_agent.py b/laya/onnx_agent.pyindex e7188784..50ccbd67 100644--- a/laya/onnx_agent.py+++ b/laya/onnx_agent.py@@ -462,6 +462,15 @@ class ONNXAgent(HookRegistry):# leave tokens no window reads at all. `README.md` and `Router.predict_long` document this# contract for both agents, so both have to honour it.ids = list(questions.keys())+ # Validate before `_to_internal`, which does none: hoisting only `_to_internal` ahead of the+ # hooks turned the ValueError `_check_question` exists to raise into+ # `AttributeError: 'NoneType' object has no attribute 'items'` -- verbatim the message its+ # own docstring says it prevents, and a regression from `main` on this path. The torch+ # `predict_long` hoists both. `laya.serve` maps ValueError to 422 and anything else to 500.+ from .agent import Agent as _Agent # deferred, as `predict_batch` does++ for qid in ids:+ _Agent._check_question(qid, questions[qid])internal = {qid: self._to_internal(questions[qid]) for qid in ids}budget, step_default, _ = window_budget(self.tok, [internal[qid] for qid in ids], max_len,head_max_len, window=window, stride=stride)diff --git a/tests/test_predict_long.py b/tests/test_predict_long.pyindex 3100fef9..219335c5 100644--- a/tests/test_predict_long.py+++ b/tests/test_predict_long.py@@ -17,7 +17,10 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))import numpy as np # noqa: E402-from laya.agent import Agent # noqa: E402+from laya.agent import Agent, _window_budget_guard++from laya import common+from laya.onnx_agent import ONNXAgent # noqa: E402PASS, FAIL = [], []@@ -63,7 +66,7 @@ def make_agent(batch_result_fn):a.tok = _Tok()a._to_internal = staticmethod(Agent._to_internal).__func__a._calls = {"system_one": 0, "batch_states": None, "system_one_kwargs": None,- "batch_kwargs": None}+ "batch_kwargs": None, "batch_size": "unset"}def _system_one(state, questions, lang=None, **controls):a._calls["system_one"] += 1@@ -73,6 +76,7 @@ def make_agent(batch_result_fn):def _predict_batch(states, questions, batch_size=None, lang=None, **controls):a._calls["batch_states"] = list(states)a._calls["batch_kwargs"] = controls+ a._calls["batch_size"] = batch_sizereturn batch_result_fn(list(states), questions)a.system_one = _system_one@@ -755,8 +759,12 @@ check_true("stride/and the caller is told the stride moved too",any("stride=90" in str(w.message) and "reduced" in str(w.message) for w in caught),[str(w.message)[:90] for w in caught])_room12 = room_for(q_many(12))-check_true("stride/the clamped scan leaves no gap", all(- b - a <= _room12 for a, b in read_spans(_clamped[1], _room12)), _clamped[1][:3])+# `read_spans` clamps every span to `start + room`, so `b - a <= room` is true of ANY input --+# executed against deliberate garbage it still passed. What the clamped scan has to guarantee is+# that nothing is left unread, which `uncovered` measures and which goes red when the stride clamp+# is mutated to `size * 2`.+check("stride/the clamped scan leaves no gap",+ uncovered(read_spans(_clamped[1], _room12), STATE_TOKENS), [])# A stride past the window the caller actually asked for is still their error.check_raises("stride/past the window the caller asked for is still refused", ValueError,lambda: scan({"a": q_many(12)}, window=40, stride=80))@@ -772,7 +780,6 @@ except ImportError: # source reverted: let the checks below go red rather tdef window_batch_cap(*_a, **_k): # type: ignore[misc]return -1-from laya.onnx_agent import ONNXAgent # noqa: E402_onnx = ONNXAgent.__new__(ONNXAgent)_onnx.tok = _Tok()@@ -804,6 +811,116 @@ check("docs/predict_long's docstring documents the cap","capped at the room the" in (Agent.predict_long.__doc__ or ""), True)+# --- findings from an adversarial review -----------------------------------------------------++# The batch cap is wired into BOTH agents and nothing pinned the wiring: replacing `cap` with+# `batch_size` at either call site, or swapping window_budget's two arguments so the cap can never+# fire, left every suite green. The cap is the OOM protection the change exists to keep.+_blow = make_agent(lambda states, questions: [+ {"model": "m", "answers": {"a": {"choice": "x", "answer_confidence": 0.5}},+ "usage": {"input_tokens": 1}} for _ in states])+_blow.predict_long(LONG, {"a": q_many(16)}) # room 24 vs a 72-token config budget: 3x blow-up+# The exact value, not "some int": returning a quarter of it, or moving _WINDOW_BATCH_BLOWUP,+# both left a plausible-looking number that an in-range assertion accepted.+check("batch cap/a blown-up scan is chunked to the un-capped scan's pass size",+ _blow._calls["batch_size"], 9)++_mild = make_agent(lambda states, questions: [+ {"model": "m", "answers": {"a": {"choice": "x", "answer_confidence": 0.5}},+ "usage": {"input_tokens": 1}} for _ in states])+_mild.predict_long(LONG, {"a": q_many(2)})+check("batch cap/a mild scan keeps the single shared pass", _mild._calls["batch_size"], None)++_explicit = make_agent(lambda states, questions: [+ {"model": "m", "answers": {"a": {"choice": "x", "answer_confidence": 0.5}},+ "usage": {"input_tokens": 1}} for _ in states])+_explicit.predict_long(LONG, {"a": q_many(16)}, batch_size=3)+check("batch cap/an explicit batch_size is honoured untouched",+ _explicit._calls["batch_size"], 3)++# Both agents must USE the cap, and the torch scan must install the budget guard. The fake agent+# replaces `predict_batch` wholesale, so no hook chain runs through it and behaviour cannot see the+# wiring; the source is what distinguishes "computed" from "computed and passed on".+_torch_src = inspect.getsource(Agent.predict_long)+_onnx_src = inspect.getsource(ONNXAgent.predict_long)+check_true("batch cap/the torch scan passes the cap to predict_batch",+ "batch_size=cap" in _torch_src, "")+check_true("batch cap/the ONNX scan passes the cap to predict_batch",+ "window_batch_cap(" in _onnx_src and "batch_size=cap" in _onnx_src, "")+# Constructing it is not installing it: removing only the hand-off left the construction in place+# and a "is it mentioned?" check green.+check_true("hook budget/the torch scan installs the budget guard",+ "_window_budget_guard(" in _torch_src and "[guard, probe]" in _torch_src, "")+# Compared on the statements, not on any mention: the comment above the call names `_to_internal`.+check_true("questions/the ONNX scan validates before _to_internal",+ _onnx_src.index("_Agent._check_question(qid")+ < _onnx_src.index("internal = {qid: self._to_internal"), "")+++# A start hook may set ctx.max_len / ctx.head_max_len, and those are what `build_sequence` uses --+# but the scan was sized from the agent's config. `widen_for_high_cardinality`, which+# docs/hooks/patterns.md ships for exactly these questions, widens the head more than max_len, so+# the real room SHRINKS: measured on the English checkpoint at 50 options, windows sized 303 against+# a real room of 253, and at 100 options 43.4% of the document reached no model -- worse than the+# 37.6% this change removes. Refusing is the honest answer; silently scanning is not.+def _widen_head_only(ctx):+ ctx.head_max_len = 60 # cfg is max_len=100 head_max_len=20, so the room shrinks+++def _widen_both(ctx):+ ctx.max_len, ctx.head_max_len = 400, 60 # widened together: the room grows+++_guard = _window_budget_guard(TOK, [Agent._to_internal(q_many(12))],+ sized=72, cfg_max_len=100, cfg_head_max_len=20)+++class _Ctx:+ def __init__(self, max_len=None, head_max_len=None):+ self.max_len, self.head_max_len = max_len, head_max_len++++def _zero_room_message():+ try:+ common.window_budget(TOK, [Agent._to_internal(q_many(24))], 100, 20,+ window=None, stride=None)+ except ValueError as exc:+ return str(exc)+ return ""++def _guard_message():+ """The text of the refusal, so the message itself is asserted rather than just the type."""+ try:+ _guard(_Ctx(100, 60))+ except ValueError as exc:+ return str(exc)+ return ""+++check_true("hook budget/an unchanged budget is allowed",+ _guard(_Ctx()) is None, "no hook")+check_true("hook budget/a budget widened together is allowed",+ _guard(_Ctx(400, 60)) is None, "room grows")+check_raises("hook budget/a hook that shrinks the room is refused", ValueError,+ lambda: _guard(_Ctx(100, 60)))+# The limitation this replaces: one oversized question aborts a whole multi-question call, because+# the budget takes min(rooms). Untested before -- `if room <= 0:` -> `if room < 0:` can never fire,+# since `state_room` clamps at 0, and every suite stayed green.+check("room/24 options leave exactly zero state tokens",+ common.state_room(TOK, Agent._to_internal(q_many(24)), 100, 20), 0)+check_raises("room/questions that fill the sequence are refused, not silently truncated", ValueError,+ lambda: common.window_budget(TOK, [Agent._to_internal(q_many(24))], 100, 20,+ window=None, stride=None))+# `room <= 0` must be the boundary: `room < 0` can never fire, since `state_room` clamps at 0, so+# the refusal would be dead code and a zero-room scan would proceed. Asserted on the message, which+# only the `<= 0` branch produces.+check_true("room/the refusal is the zero-room branch, not an incidental error",+ "no window can carry any of it" in _zero_room_message(), _zero_room_message())+check_true("hook budget/the refusal names both budgets and both rooms",+ all(t in _guard_message() for t in ("max_len=100", "head_max_len=60", "sized for 72")),+ _guard_message())+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL:print(" FAIL " + f)