Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(predict_long): refuse a scan a start hook has re-budgeted, and restore ONNX validation

Adeshina Lasisi20cb8b1de53d
diff --git a/laya/agent.py b/laya/agent.py
index f8229fbb..69f8d34b 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -42,6 +42,7 @@ from .common import (
temp_bucket,
unpermute_probs,
window_batch_cap,
+ state_room,
window_budget,
)
from .confidence import check_min_confidence, flag_low_confidence
@@ -282,10 +283,52 @@ def _start_evidence():
return probe, evidence
+def _window_budget_guard(tok, internal, sized, cfg_max_len, cfg_head_max_len):
+ """A start hook that refuses a scan whose windows no longer fit the budget in force.
+
+ `predict_long` sizes its windows from the agent's config, but a start hook may set
+ `ctx.max_len` / `ctx.head_max_len`, and those are what `build_sequence` uses. When the hook
+ widens the head more than it widens `max_len` -- which `widen_for_high_cardinality` in
+ `docs/hooks/patterns.md` does, and it is documented for exactly the high-cardinality questions
+ that make windowing necessary -- the real room SHRINKS, every window is re-truncated on the way
+ in, and once the stride exceeds the real room consecutive windows stop touching. Measured on the
+ English checkpoint with that hook at 50 options: windows sized 303, room actually 253; at 100
+ options the reviewer measured 43.4% of a document reaching no model, against the 37.6% this
+ change exists to remove. Silently worse than not windowing at all.
+
+ Sizing after the hook chain would mean dispatching it before the windows exist, which changes
+ when hooks run and what they see. Refusing is the honest alternative: the scan cannot honour a
+ budget it was not sized for, and a caller who needs both can pass `window=`/`stride=` explicitly
+ or drive `predict_batch` itself. This runs in the start chain, so it aborts before any forward
+ pass rather than after one.
+ """
+ def guard(ctx):
+ eff_max_len = ctx.max_len if ctx.max_len is not None else cfg_max_len
+ eff_head = ctx.head_max_len if ctx.head_max_len is not None else cfg_head_max_len
+ if eff_max_len == cfg_max_len and eff_head == cfg_head_max_len:
+ return
+ room = min(state_room(tok, q, eff_max_len, eff_head) for q in internal)
+ if room >= sized:
+ return # the budget moved, but not in a direction that truncates
+ raise ValueError(
+ "predict_long: a start hook set the token budget to max_len=%d head_max_len=%d, which "
+ "leaves %d state tokens per window, but the scan was sized for %d from the agent's "
+ "config (max_len=%d head_max_len=%d). Every window would be re-truncated and parts of "
+ "the document would reach no model. Pass window=/stride= explicitly, or call "
+ "predict_batch directly, if you need both."
+ % (eff_max_len, eff_head, room, sized, cfg_max_len, cfg_head_max_len))
+
+ return guard
+
+
def _with_start_probe(hook_kwargs, probe):
- """`hook_kwargs` with `probe` appended after the caller's own start hooks."""
+ """`hook_kwargs` with `probe` appended after the caller's own start hooks.
+
+ `probe` may be one hook or a list of them, appended in order.
+ """
kwargs = dict(hook_kwargs)
- kwargs["on_predict_start"] = list(_as_sequence(hook_kwargs.get("on_predict_start"))) + [probe]
+ extra = list(probe) if isinstance(probe, (list, tuple)) else [probe]
+ kwargs["on_predict_start"] = list(_as_sequence(hook_kwargs.get("on_predict_start"))) + extra
return kwargs
def _option_logits(logits, items, offset):
"""Raw per-option logits, the rows `_decode_answers` divides by temperature.
@@ -1435,6 +1478,8 @@ class Agent(HookRegistry):
i += step
probe, evidence = _start_evidence()
+ guard = _window_budget_guard(self.tok, [internal[qid] for qid in ids], budget,
+ max_len, head_max_len)
# `list(windows)`, not `windows`: `ctx.states` is the list the hook receives, so a hook that
# mutates it in place (`append`, `sort`) would otherwise also grow the split this call
# attributes answers to, and the counts would agree while `starts` no longer lined up.
@@ -1443,7 +1488,7 @@ class Agent(HookRegistry):
cap = window_batch_cap(len(windows), budget, max(64, max_len - head_max_len - 8),
batch_size)
results = self.predict_batch(list(windows), questions, batch_size=cap, lang=lang,
- **_with_start_probe(hook_kwargs, probe))
+ **_with_start_probe(hook_kwargs, [guard, probe]))
if evidence["answered"]:
# The hook replaced the call before any window was scored. Aggregating over its payload
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py
index e7188784..50ccbd67 100644
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@ -462,6 +462,15 @@ class ONNXAgent(HookRegistry):
# leave tokens no window reads at all. `README.md` and `Router.predict_long` document this
# contract for both agents, so both have to honour it.
ids = list(questions.keys())
+ # Validate before `_to_internal`, which does none: hoisting only `_to_internal` ahead of the
+ # hooks turned the ValueError `_check_question` exists to raise into
+ # `AttributeError: 'NoneType' object has no attribute 'items'` -- verbatim the message its
+ # own docstring says it prevents, and a regression from `main` on this path. The torch
+ # `predict_long` hoists both. `laya.serve` maps ValueError to 422 and anything else to 500.
+ from .agent import Agent as _Agent # deferred, as `predict_batch` does
+
+ for qid in ids:
+ _Agent._check_question(qid, questions[qid])
internal = {qid: self._to_internal(questions[qid]) for qid in ids}
budget, step_default, _ = window_budget(self.tok, [internal[qid] for qid in ids], max_len,
head_max_len, window=window, stride=stride)
diff --git a/tests/test_predict_long.py b/tests/test_predict_long.py
index 3100fef9..219335c5 100644
--- a/tests/test_predict_long.py
+++ b/tests/test_predict_long.py
@@ -17,7 +17,10 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import numpy as np # noqa: E402
-from laya.agent import Agent # noqa: E402
+from laya.agent import Agent, _window_budget_guard
+
+from laya import common
+from laya.onnx_agent import ONNXAgent # noqa: E402
PASS, FAIL = [], []
@@ -63,7 +66,7 @@ def make_agent(batch_result_fn):
a.tok = _Tok()
a._to_internal = staticmethod(Agent._to_internal).__func__
a._calls = {"system_one": 0, "batch_states": None, "system_one_kwargs": None,
- "batch_kwargs": None}
+ "batch_kwargs": None, "batch_size": "unset"}
def _system_one(state, questions, lang=None, **controls):
a._calls["system_one"] += 1
@@ -73,6 +76,7 @@ def make_agent(batch_result_fn):
def _predict_batch(states, questions, batch_size=None, lang=None, **controls):
a._calls["batch_states"] = list(states)
a._calls["batch_kwargs"] = controls
+ a._calls["batch_size"] = batch_size
return batch_result_fn(list(states), questions)
a.system_one = _system_one
@@ -755,8 +759,12 @@ check_true("stride/and the caller is told the stride moved too",
any("stride=90" in str(w.message) and "reduced" in str(w.message) for w in caught),
[str(w.message)[:90] for w in caught])
_room12 = room_for(q_many(12))
-check_true("stride/the clamped scan leaves no gap", all(
- b - a <= _room12 for a, b in read_spans(_clamped[1], _room12)), _clamped[1][:3])
+# `read_spans` clamps every span to `start + room`, so `b - a <= room` is true of ANY input --
+# executed against deliberate garbage it still passed. What the clamped scan has to guarantee is
+# that nothing is left unread, which `uncovered` measures and which goes red when the stride clamp
+# is mutated to `size * 2`.
+check("stride/the clamped scan leaves no gap",
+ uncovered(read_spans(_clamped[1], _room12), STATE_TOKENS), [])
# A stride past the window the caller actually asked for is still their error.
check_raises("stride/past the window the caller asked for is still refused", ValueError,
lambda: scan({"a": q_many(12)}, window=40, stride=80))
@@ -772,7 +780,6 @@ except ImportError: # source reverted: let the checks below go red rather t
def window_batch_cap(*_a, **_k): # type: ignore[misc]
return -1
-from laya.onnx_agent import ONNXAgent # noqa: E402
_onnx = ONNXAgent.__new__(ONNXAgent)
_onnx.tok = _Tok()
@@ -804,6 +811,116 @@ check("docs/predict_long's docstring documents the cap",
"capped at the room the" in (Agent.predict_long.__doc__ or ""), True)
+# --- findings from an adversarial review -----------------------------------------------------
+
+# The batch cap is wired into BOTH agents and nothing pinned the wiring: replacing `cap` with
+# `batch_size` at either call site, or swapping window_budget's two arguments so the cap can never
+# fire, left every suite green. The cap is the OOM protection the change exists to keep.
+_blow = make_agent(lambda states, questions: [
+ {"model": "m", "answers": {"a": {"choice": "x", "answer_confidence": 0.5}},
+ "usage": {"input_tokens": 1}} for _ in states])
+_blow.predict_long(LONG, {"a": q_many(16)}) # room 24 vs a 72-token config budget: 3x blow-up
+# The exact value, not "some int": returning a quarter of it, or moving _WINDOW_BATCH_BLOWUP,
+# both left a plausible-looking number that an in-range assertion accepted.
+check("batch cap/a blown-up scan is chunked to the un-capped scan's pass size",
+ _blow._calls["batch_size"], 9)
+
+_mild = make_agent(lambda states, questions: [
+ {"model": "m", "answers": {"a": {"choice": "x", "answer_confidence": 0.5}},
+ "usage": {"input_tokens": 1}} for _ in states])
+_mild.predict_long(LONG, {"a": q_many(2)})
+check("batch cap/a mild scan keeps the single shared pass", _mild._calls["batch_size"], None)
+
+_explicit = make_agent(lambda states, questions: [
+ {"model": "m", "answers": {"a": {"choice": "x", "answer_confidence": 0.5}},
+ "usage": {"input_tokens": 1}} for _ in states])
+_explicit.predict_long(LONG, {"a": q_many(16)}, batch_size=3)
+check("batch cap/an explicit batch_size is honoured untouched",
+ _explicit._calls["batch_size"], 3)
+
+# Both agents must USE the cap, and the torch scan must install the budget guard. The fake agent
+# replaces `predict_batch` wholesale, so no hook chain runs through it and behaviour cannot see the
+# wiring; the source is what distinguishes "computed" from "computed and passed on".
+_torch_src = inspect.getsource(Agent.predict_long)
+_onnx_src = inspect.getsource(ONNXAgent.predict_long)
+check_true("batch cap/the torch scan passes the cap to predict_batch",
+ "batch_size=cap" in _torch_src, "")
+check_true("batch cap/the ONNX scan passes the cap to predict_batch",
+ "window_batch_cap(" in _onnx_src and "batch_size=cap" in _onnx_src, "")
+# Constructing it is not installing it: removing only the hand-off left the construction in place
+# and a "is it mentioned?" check green.
+check_true("hook budget/the torch scan installs the budget guard",
+ "_window_budget_guard(" in _torch_src and "[guard, probe]" in _torch_src, "")
+# Compared on the statements, not on any mention: the comment above the call names `_to_internal`.
+check_true("questions/the ONNX scan validates before _to_internal",
+ _onnx_src.index("_Agent._check_question(qid")
+ < _onnx_src.index("internal = {qid: self._to_internal"), "")
+
+
+# A start hook may set ctx.max_len / ctx.head_max_len, and those are what `build_sequence` uses --
+# but the scan was sized from the agent's config. `widen_for_high_cardinality`, which
+# docs/hooks/patterns.md ships for exactly these questions, widens the head more than max_len, so
+# the real room SHRINKS: measured on the English checkpoint at 50 options, windows sized 303 against
+# a real room of 253, and at 100 options 43.4% of the document reached no model -- worse than the
+# 37.6% this change removes. Refusing is the honest answer; silently scanning is not.
+def _widen_head_only(ctx):
+ ctx.head_max_len = 60 # cfg is max_len=100 head_max_len=20, so the room shrinks
+
+
+def _widen_both(ctx):
+ ctx.max_len, ctx.head_max_len = 400, 60 # widened together: the room grows
+
+
+_guard = _window_budget_guard(TOK, [Agent._to_internal(q_many(12))],
+ sized=72, cfg_max_len=100, cfg_head_max_len=20)
+
+
+class _Ctx:
+ def __init__(self, max_len=None, head_max_len=None):
+ self.max_len, self.head_max_len = max_len, head_max_len
+
+
+
+def _zero_room_message():
+ try:
+ common.window_budget(TOK, [Agent._to_internal(q_many(24))], 100, 20,
+ window=None, stride=None)
+ except ValueError as exc:
+ return str(exc)
+ return ""
+
+def _guard_message():
+ """The text of the refusal, so the message itself is asserted rather than just the type."""
+ try:
+ _guard(_Ctx(100, 60))
+ except ValueError as exc:
+ return str(exc)
+ return ""
+
+
+check_true("hook budget/an unchanged budget is allowed",
+ _guard(_Ctx()) is None, "no hook")
+check_true("hook budget/a budget widened together is allowed",
+ _guard(_Ctx(400, 60)) is None, "room grows")
+check_raises("hook budget/a hook that shrinks the room is refused", ValueError,
+ lambda: _guard(_Ctx(100, 60)))
+# The limitation this replaces: one oversized question aborts a whole multi-question call, because
+# the budget takes min(rooms). Untested before -- `if room <= 0:` -> `if room < 0:` can never fire,
+# since `state_room` clamps at 0, and every suite stayed green.
+check("room/24 options leave exactly zero state tokens",
+ common.state_room(TOK, Agent._to_internal(q_many(24)), 100, 20), 0)
+check_raises("room/questions that fill the sequence are refused, not silently truncated", ValueError,
+ lambda: common.window_budget(TOK, [Agent._to_internal(q_many(24))], 100, 20,
+ window=None, stride=None))
+# `room <= 0` must be the boundary: `room < 0` can never fire, since `state_room` clamps at 0, so
+# the refusal would be dead code and a zero-room scan would proceed. Asserted on the message, which
+# only the `<= 0` branch produces.
+check_true("room/the refusal is the zero-room branch, not an incidental error",
+ "no window can carry any of it" in _zero_room_message(), _zero_room_message())
+check_true("hook budget/the refusal names both budgets and both rooms",
+ all(t in _guard_message() for t in ("max_len=100", "head_max_len=60", "sized for 72")),
+ _guard_message())
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL " + f)