Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(predict_long): the budget check must not be a hook, and must cover both agents

Adeshina Lasisiee3f9d43f6f9
diff --git a/laya/agent.py b/laya/agent.py
index 69f8d34b..f336e44a 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -267,7 +267,8 @@ def _start_evidence():
inference, `states` is a snapshot of the states that reached it (`None` if the probe never ran,
which means `predict_batch` was replaced and no hook chain was dispatched).
"""
- evidence = {"answered": False, "states": None, "question_types": None}
+ evidence = {"answered": False, "states": None, "question_types": None,
+ "max_len": None, "head_max_len": None, "questions": None}
def probe(ctx):
evidence["answered"] = ctx.results is not None
@@ -279,46 +280,65 @@ def _start_evidence():
qid: qdef.get("type") if isinstance(qdef, dict) else None
for qid, qdef in ctx.questions.items()
}
+ # Also what the budget and the questions ACTUALLY are once the chain has run, for
+ # `_check_scan_budget`. Recorded, never judged here: a hook that raises is governed by the
+ # caller's `hooks_raise`/`hooks_timeout`, so a correctness check that raises from inside the
+ # chain can be switched off by a policy meant for third-party telemetry. Measured: with
+ # `hooks_raise=False` -- what `docs/hooks/tracing.md` recommends -- a refusal became a
+ # RuntimeWarning blaming `_StartAdapter`, and the scan proceeded.
+ evidence["max_len"] = ctx.max_len
+ evidence["head_max_len"] = ctx.head_max_len
+ evidence["questions"] = dict(ctx.questions) if ctx.questions else {}
return probe, evidence
-def _window_budget_guard(tok, internal, sized, cfg_max_len, cfg_head_max_len):
- """A start hook that refuses a scan whose windows no longer fit the budget in force.
-
- `predict_long` sizes its windows from the agent's config, but a start hook may set
- `ctx.max_len` / `ctx.head_max_len`, and those are what `build_sequence` uses. When the hook
- widens the head more than it widens `max_len` -- which `widen_for_high_cardinality` in
- `docs/hooks/patterns.md` does, and it is documented for exactly the high-cardinality questions
- that make windowing necessary -- the real room SHRINKS, every window is re-truncated on the way
- in, and once the stride exceeds the real room consecutive windows stop touching. Measured on the
- English checkpoint with that hook at 50 options: windows sized 303, room actually 253; at 100
- options the reviewer measured 43.4% of a document reaching no model, against the 37.6% this
- change exists to remove. Silently worse than not windowing at all.
-
- Sizing after the hook chain would mean dispatching it before the windows exist, which changes
- when hooks run and what they see. Refusing is the honest alternative: the scan cannot honour a
- budget it was not sized for, and a caller who needs both can pass `window=`/`stride=` explicitly
- or drive `predict_batch` itself. This runs in the start chain, so it aborts before any forward
- pass rather than after one.
+def _check_scan_budget(agent, evidence, sized, cfg_max_len, cfg_head_max_len):
+ """Raise when the budget or questions in force leave less room than the scan was sized for.
+
+ `predict_long` sizes its windows from the agent's config, before any hook has run. A start hook
+ may then set `ctx.max_len`/`ctx.head_max_len`, or rewrite `ctx.questions` -- both documented
+ powers -- and `build_sequence` uses whatever it finds. When the head widens faster than
+ `max_len`, or the questions get more options, the real room SHRINKS: every window is
+ re-truncated on the way in, and once the stride exceeds the real room consecutive windows stop
+ touching. Measured on the English checkpoint with `widen_for_high_cardinality` from
+ `docs/hooks/patterns.md` at 50 options, windows sized 303 against a room of 253; at 100 options
+ a reviewer measured 43.4% of a document reaching no model, against the 37.6% this change exists
+ to remove. Silently worse than not windowing at all.
+
+ Called by `predict_long` AFTER the chain has run, not raised from inside it. A hook that raises
+ is subject to the caller's `hooks_raise` and `hooks_timeout`, so the first version of this check
+ could be switched off by `hooks_raise=False` -- which `docs/hooks/tracing.md` recommends -- and
+ could blow a `hooks_timeout` the caller set for their own hooks. The cost is that the forward
+ pass has already happened when this fires: a misconfigured scan is refused rather than answered
+ wrongly, but it is not refused for free. Sizing the windows after the chain instead would change
+ when hooks run and what they see, which is a larger change than this one.
"""
- def guard(ctx):
- eff_max_len = ctx.max_len if ctx.max_len is not None else cfg_max_len
- eff_head = ctx.head_max_len if ctx.head_max_len is not None else cfg_head_max_len
- if eff_max_len == cfg_max_len and eff_head == cfg_head_max_len:
- return
- room = min(state_room(tok, q, eff_max_len, eff_head) for q in internal)
- if room >= sized:
- return # the budget moved, but not in a direction that truncates
- raise ValueError(
- "predict_long: a start hook set the token budget to max_len=%d head_max_len=%d, which "
- "leaves %d state tokens per window, but the scan was sized for %d from the agent's "
- "config (max_len=%d head_max_len=%d). Every window would be re-truncated and parts of "
- "the document would reach no model. Pass window=/stride= explicitly, or call "
- "predict_batch directly, if you need both."
- % (eff_max_len, eff_head, room, sized, cfg_max_len, cfg_head_max_len))
-
- return guard
+ if evidence.get("answered"):
+ return # a hook answered the document; no window was scored
+ questions = evidence.get("questions")
+ if not questions:
+ return # nothing to fit, as `window_budget` also concludes
+ eff_max_len = evidence.get("max_len")
+ eff_head = evidence.get("head_max_len")
+ eff_max_len = cfg_max_len if eff_max_len is None else eff_max_len
+ eff_head = cfg_head_max_len if eff_head is None else eff_head
+ try:
+ internal = [agent._to_internal(q) for q in questions.values()]
+ except Exception:
+ return # malformed questions are the validator's to report
+ if not internal:
+ return
+ room = min(state_room(agent.tok, q, eff_max_len, eff_head) for q in internal)
+ if room >= sized:
+ return
+ raise ValueError(
+ "predict_long: after the start hooks ran, the questions and token budget (max_len=%d "
+ "head_max_len=%d) leave %d state tokens per window, but the scan was sized for %d from the "
+ "agent's config (max_len=%d head_max_len=%d). Every window would be re-truncated and parts "
+ "of the document would reach no model. Pass window=/stride= explicitly, or call "
+ "predict_batch directly, if you need a start hook to change either."
+ % (eff_max_len, eff_head, room, sized, cfg_max_len, cfg_head_max_len))
def _with_start_probe(hook_kwargs, probe):
@@ -1462,6 +1482,11 @@ class Agent(HookRegistry):
# Why 0 for a hook answer here: the state did fit one window, but no window was scored,
# which is the same fact the multi-window path reports as 0. Reading 1 would make a
# cached answer and a served answer agree on how much of the input the model saw.
+ # The same budget check as the multi-window path. Without it a document short enough to
+ # fit one window was silently truncated by a re-budgeting hook and still reported
+ # `windows: 1`, i.e. "the model read all of it" -- measured, 138 of 240 state tokens
+ # never reached the model, while a longer document on the identical input hard-failed.
+ _check_scan_budget(self, evidence, budget, max_len, head_max_len)
single["usage"] = {**(single.get("usage") or {}), "windows": 0 if evidence["answered"] else 1}
return single
@@ -1478,8 +1503,6 @@ class Agent(HookRegistry):
i += step
probe, evidence = _start_evidence()
- guard = _window_budget_guard(self.tok, [internal[qid] for qid in ids], budget,
- max_len, head_max_len)
# `list(windows)`, not `windows`: `ctx.states` is the list the hook receives, so a hook that
# mutates it in place (`append`, `sort`) would otherwise also grow the split this call
# attributes answers to, and the counts would agree while `starts` no longer lined up.
@@ -1488,7 +1511,8 @@ class Agent(HookRegistry):
cap = window_batch_cap(len(windows), budget, max(64, max_len - head_max_len - 8),
batch_size)
results = self.predict_batch(list(windows), questions, batch_size=cap, lang=lang,
- **_with_start_probe(hook_kwargs, [guard, probe]))
+ **_with_start_probe(hook_kwargs, probe))
+ _check_scan_budget(self, evidence, budget, max_len, head_max_len)
if evidence["answered"]:
# The hook replaced the call before any window was scored. Aggregating over its payload
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py
index 50ccbd67..23ec2ccd 100644
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@ -440,7 +440,7 @@ class ONNXAgent(HookRegistry):
in `Agent.predict_long`: `truncated` is a window count and `truncated_questions` is the
last window's list, so `truncated` can be above 0 while the list is empty.
"""
- from .agent import _start_evidence, _with_start_probe
+ from .agent import _check_scan_budget, _start_evidence, _with_start_probe
from .hooks import aggregate_usage
hook_kwargs = {"hooks": hooks, "on_predict_start": on_predict_start,
@@ -486,6 +486,11 @@ class ONNXAgent(HookRegistry):
probe, evidence = _start_evidence()
single = dict(self.system_one(state, questions, lang=lang,
**_with_start_probe(hook_kwargs, probe)))
+ # The same budget check as the multi-window path. Without it a document short enough to
+ # fit one window was silently truncated by a re-budgeting hook and still reported
+ # `windows: 1`, i.e. "the model read all of it" -- measured, 138 of 240 state tokens
+ # never reached the model, while a longer document on the identical input hard-failed.
+ _check_scan_budget(self, evidence, budget, max_len, head_max_len)
single["usage"] = {**(single.get("usage") or {}), "windows": 0 if evidence["answered"] else 1}
return single
@@ -509,6 +514,12 @@ class ONNXAgent(HookRegistry):
batch_size)
results = self.predict_batch(list(windows), questions, batch_size=cap, lang=lang,
**_with_start_probe(hook_kwargs, probe))
+ # Same check as the torch agent, for the same reason and on the same contract: a start hook
+ # re-budgets an ONNX scan exactly as it re-budgets a torch one (`predict_batch` applies
+ # `ctx.max_len`/`ctx.head_max_len` identically), so leaving it off here meant the bug was
+ # fully live on this path while the other agent refused the identical input -- measured,
+ # 34.6% of a document reaching no model.
+ _check_scan_budget(self, evidence, budget, max_len, head_max_len)
if evidence["answered"]:
# A hook answered the document before any window was scored: pass that answer
diff --git a/tests/test_predict_long.py b/tests/test_predict_long.py
index 219335c5..6ea2a7ff 100644
--- a/tests/test_predict_long.py
+++ b/tests/test_predict_long.py
@@ -17,7 +17,8 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import numpy as np # noqa: E402
-from laya.agent import Agent, _window_budget_guard
+import laya.agent as agent_mod
+from laya.agent import Agent, _check_scan_budget, _start_evidence
from laya import common
from laya.onnx_agent import ONNXAgent # noqa: E402
@@ -847,79 +848,133 @@ check_true("batch cap/the torch scan passes the cap to predict_batch",
"batch_size=cap" in _torch_src, "")
check_true("batch cap/the ONNX scan passes the cap to predict_batch",
"window_batch_cap(" in _onnx_src and "batch_size=cap" in _onnx_src, "")
-# Constructing it is not installing it: removing only the hand-off left the construction in place
-# and a "is it mentioned?" check green.
-check_true("hook budget/the torch scan installs the budget guard",
- "_window_budget_guard(" in _torch_src and "[guard, probe]" in _torch_src, "")
# Compared on the statements, not on any mention: the comment above the call names `_to_internal`.
check_true("questions/the ONNX scan validates before _to_internal",
_onnx_src.index("_Agent._check_question(qid")
< _onnx_src.index("internal = {qid: self._to_internal"), "")
-# A start hook may set ctx.max_len / ctx.head_max_len, and those are what `build_sequence` uses --
-# but the scan was sized from the agent's config. `widen_for_high_cardinality`, which
-# docs/hooks/patterns.md ships for exactly these questions, widens the head more than max_len, so
-# the real room SHRINKS: measured on the English checkpoint at 50 options, windows sized 303 against
-# a real room of 253, and at 100 options 43.4% of the document reached no model -- worse than the
-# 37.6% this change removes. Refusing is the honest answer; silently scanning is not.
-def _widen_head_only(ctx):
- ctx.head_max_len = 60 # cfg is max_len=100 head_max_len=20, so the room shrinks
-
-
-def _widen_both(ctx):
- ctx.max_len, ctx.head_max_len = 400, 60 # widened together: the room grows
+# A start hook may set ctx.max_len / ctx.head_max_len, or rewrite ctx.questions -- both documented
+# powers -- and `build_sequence` uses whatever it finds, while the scan was sized from the config
+# before any hook ran. `widen_for_high_cardinality`, which docs/hooks/patterns.md ships for exactly
+# these questions, widens the head faster than max_len, so the real room SHRINKS: 303 sized against
+# 253 real at 50 options on the English checkpoint, and 43.4% of a document unread at 100.
+#
+# The first version of this was a start hook that RAISED, and three ways of defeating it all left
+# the suite green: `hooks_raise=False` -- what docs/hooks/tracing.md recommends -- downgraded the
+# refusal to a RuntimeWarning and the scan proceeded; a hook that rewrote ctx.questions instead of
+# the budget was invisible; and `sized=0` at the call site made it unable to fire while the
+# source-text assertions still matched. It is now a plain function `predict_long` calls itself, so
+# no hook policy governs it, and both halves are checked below: what it decides, and that it is
+# actually called with the budget the scan used.
+_CFG = (100, 20) # make_agent's config; budget = max(64, 100-20-8) = 72
-_guard = _window_budget_guard(TOK, [Agent._to_internal(q_many(12))],
- sized=72, cfg_max_len=100, cfg_head_max_len=20)
+def _evidence(max_len=None, head_max_len=None, questions=None, answered=False):
+ return {"answered": answered, "states": None, "max_len": max_len,
+ "head_max_len": head_max_len,
+ "questions": {"a": q_many(2)} if questions is None else questions}
-class _Ctx:
- def __init__(self, max_len=None, head_max_len=None):
- self.max_len, self.head_max_len = max_len, head_max_len
+_probe_agent = make_agent(lambda states, qs: [])
+def _scan_raw(questions=None, **kw):
+ a = make_agent(lambda states, qs: [])
+ return a.predict_long(LONG, questions or {"a": q_many(2)}, **kw)
-def _zero_room_message():
- try:
- common.window_budget(TOK, [Agent._to_internal(q_many(24))], 100, 20,
- window=None, stride=None)
- except ValueError as exc:
- return str(exc)
- return ""
-def _guard_message():
- """The text of the refusal, so the message itself is asserted rather than just the type."""
+def _budget_message():
try:
- _guard(_Ctx(100, 60))
+ _check_scan_budget(_probe_agent, _evidence(head_max_len=60), 72, *_CFG)
except ValueError as exc:
return str(exc)
return ""
+check_raises("hook budget/a hook that shrinks the room is refused", ValueError,
+ lambda: _check_scan_budget(_probe_agent, _evidence(head_max_len=60), 72, *_CFG))
+check_raises("hook budget/rewriting ctx.questions is caught too", ValueError,
+ lambda: _check_scan_budget(_probe_agent, _evidence(questions={"a": q_many(16)}),
+ 72, *_CFG))
check_true("hook budget/an unchanged budget is allowed",
- _guard(_Ctx()) is None, "no hook")
+ _check_scan_budget(_probe_agent, _evidence(), 72, *_CFG) is None, "no hook")
check_true("hook budget/a budget widened together is allowed",
- _guard(_Ctx(400, 60)) is None, "room grows")
-check_raises("hook budget/a hook that shrinks the room is refused", ValueError,
- lambda: _guard(_Ctx(100, 60)))
-# The limitation this replaces: one oversized question aborts a whole multi-question call, because
-# the budget takes min(rooms). Untested before -- `if room <= 0:` -> `if room < 0:` can never fire,
-# since `state_room` clamps at 0, and every suite stayed green.
-check("room/24 options leave exactly zero state tokens",
- common.state_room(TOK, Agent._to_internal(q_many(24)), 100, 20), 0)
-check_raises("room/questions that fill the sequence are refused, not silently truncated", ValueError,
- lambda: common.window_budget(TOK, [Agent._to_internal(q_many(24))], 100, 20,
- window=None, stride=None))
-# `room <= 0` must be the boundary: `room < 0` can never fire, since `state_room` clamps at 0, so
-# the refusal would be dead code and a zero-room scan would proceed. Asserted on the message, which
-# only the `<= 0` branch produces.
-check_true("room/the refusal is the zero-room branch, not an incidental error",
- "no window can carry any of it" in _zero_room_message(), _zero_room_message())
+ _check_scan_budget(_probe_agent, _evidence(400, 60), 72, *_CFG) is None, "room grows")
+check_true("hook budget/a hook that answered the document is not refused",
+ _check_scan_budget(_probe_agent, _evidence(head_max_len=60, answered=True),
+ 72, *_CFG) is None, "cache hit")
+check_true("hook budget/no questions is not a crash",
+ _check_scan_budget(_probe_agent, _evidence(head_max_len=60, questions={}),
+ 72, *_CFG) is None, "empty questions")
check_true("hook budget/the refusal names both budgets and both rooms",
- all(t in _guard_message() for t in ("max_len=100", "head_max_len=60", "sized for 72")),
- _guard_message())
+ all(t in _budget_message() for t in ("max_len=100", "head_max_len=60", "sized for 72")),
+ _budget_message())
+
+# ... and that predict_long actually calls it, with the budget it sized the windows to. This is what
+# `sized=0` walked past: the symbol was present and the call site spelled right, but the value made
+# the check inert.
+_seen_budget = []
+_real_check = agent_mod._check_scan_budget
+try:
+ agent_mod._check_scan_budget = lambda ag, ev, sized, ml, hm: _seen_budget.append((sized, ml, hm))
+ _wired = make_agent(lambda states, qs: [
+ {"model": "m", "answers": {"a": {"choice": "x", "answer_confidence": 0.5}},
+ "usage": {"input_tokens": 1}} for _ in states])
+ _wired.predict_long(LONG, {"a": q_many(2)})
+finally:
+ agent_mod._check_scan_budget = _real_check
+check("hook budget/the torch scan checks its budget exactly once", len(_seen_budget), 1)
+# The room is the SMALLEST any question leaves, which is why `window_budget` takes a min. No test
+# put two questions of different cardinality through the checker, so `min` -> `max` was green.
+# At the unchanged budget these two leave rooms of 76 and 24 against a scan sized for 72, so `min`
+# refuses and `max` does not -- which is the whole reason `window_budget` takes a min. Collapsing
+# them under a widened head (both 36) makes the mutant survive, which is how it slipped through.
+check_raises("hook budget/the smallest room wins across questions", ValueError,
+ lambda: _check_scan_budget(
+ _probe_agent, _evidence(questions={"a": q_many(2), "b": q_many(16)}), 72, *_CFG))
+
+# What the probe RECORDS is half the check: synthesising an evidence dict in the tests above leaves
+# the recording itself unpinned, and dropping either field made the checker silently inert.
+_rec_probe, _rec = _start_evidence()
+
+
+class _RecCtx:
+ results = None
+ states = ["s"]
+ max_len = 400
+ head_max_len = 60
+ questions = {"a": q_many(2)}
+
+
+_rec_probe(_RecCtx())
+check("probe/records the budget in force", (_rec["max_len"], _rec["head_max_len"]), (400, 60))
+check("probe/records the questions in force", list(_rec["questions"]), ["a"])
+check_true("probe/copies the questions rather than aliasing them",
+ _rec["questions"] is not _RecCtx.questions, "")
+
+# Validation before `_to_internal` on the torch agent too. The ONNX side is pinned by a source-order
+# check; deleting the torch loop was green.
+check_raises("questions/the torch scan validates before _to_internal", ValueError,
+ lambda: _scan_raw(questions={"a": {"type": "choice", "instructions": "?",
+ "criteria": None}}))
+
+# Both agents, not one. The ONNX scan had no check at all while the torch one refused the identical
+# input -- measured, 34.6% of a document reaching no model -- and the source comment two screens up
+# says the contract binds both agents.
+_onnx_src = inspect.getsource(ONNXAgent.predict_long)
+check_true("hook budget/the ONNX scan checks its budget too",
+ "_check_scan_budget(self, evidence, budget, max_len, head_max_len)" in _onnx_src, "")
+check("hook budget/both agents check it on the single-window path too",
+ (inspect.getsource(Agent.predict_long).count("_check_scan_budget("),
+ _onnx_src.count("_check_scan_budget(")), (2, 2))
+
+check("hook budget/it is handed the window size and the config it sized from",
+ _seen_budget[0] if _seen_budget else None,
+ (window_budget(TOK, [Agent._to_internal(q_many(2))], 100, 20,
+ window=None, stride=None)[0], 100, 20))
+
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL: