Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(agent): per-call hooks on predict_long, and no window for a skipped answer

Aashish70367cd96bf8
diff --git a/README.md b/README.md
index 54c20cf9..c339df94 100644
--- a/README.md
+++ b/README.md
@@ -570,12 +570,18 @@ per question:
```python
result = agent.predict_long(state, questions) # windows the state, one result back
result = agent.predict_long(state, questions, window=256) # smaller window isolates a localized span
+result = agent.predict_long(state, questions, hooks=[AuditLog()]) # the scan, instrumented
```
- `noul` takes the strongest window (the statement holds if any window supports it).
- `choice` / `score` take the most-confident window — averaging over a long, mostly-neutral
document lets the neutral majority out-vote the one window that saw the deciding span.
- A state that already fits one window is passed straight to `system_one` (identical output).
+- Hooks wrap the inference that answers the state, so on a scanned document `on_predict_start`
+ fires once with `ctx.states` holding the decoded windows, not the state you passed in (it was
+ tokenized to produce them). A start hook that answers with `ctx.skip(...)` is answering the
+ whole document: its result comes back with no `answer["window"]` and `usage["windows"]` at 0,
+ because no window scored it.
A smaller `window` isolates a short deciding span better (it becomes a larger fraction of its
window); the default (`max_len - head_max_len`) favors context and throughput. Output shape matches
diff --git a/docs/hooks/api.md b/docs/hooks/api.md
index 8817e666..fc14b733 100644
--- a/docs/hooks/api.md
+++ b/docs/hooks/api.md
@@ -229,11 +229,22 @@ agent.system_one(state, questions,
hooks=None, on_predict_start=None, on_predict_end=None, hooks_raise=None,
hooks_timeout=None, max_len=None, head_max_len=None)
+agent.predict_long(state, questions, window=None, stride=None, aggregate="auto",
+ batch_size=None, lang=None,
+ hooks=None, on_predict_start=None, on_predict_end=None, hooks_raise=None,
+ hooks_timeout=None)
+
agent.predict(...) # alias of system_one
```
- `hooks_raise` and `hooks_timeout` on a per-call method default to `None`, meaning "use the instance value".
- `hooks_concurrent` is instance-level only.
+- On `predict_long` the hooks wrap the inference that answers the state, which for a document
+ needing several windows is the single shared `predict_batch` over them: `on_predict_start` fires
+ once, and `ctx.states` holds the decoded window texts in scan order rather than the caller's
+ state, which was tokenized to produce them. A start hook that answers with `ctx.skip(...)` is
+ answering the document, so its one result is returned as-is with no `window` key and
+ `usage["windows"]` at `0` -- no window scored it.
### Router
diff --git a/laya/agent.py b/laya/agent.py
index 1b58de6f..5cc8d044 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -956,7 +956,10 @@ class Agent(HookRegistry):
def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]],
window: Optional[int] = None, stride: Optional[int] = None,
aggregate: str = "auto", batch_size: Optional[int] = None,
- lang: Optional[str] = None) -> Dict[str, Any]:
+ lang: Optional[str] = None,
+ hooks=None, on_predict_start=None, on_predict_end=None,
+ hooks_raise: Optional[bool] = None,
+ hooks_timeout: Optional[float] = None) -> Dict[str, Any]:
"""Evaluate questions over a state longer than the context window, scanning it in
overlapping windows and aggregating per question.
@@ -980,6 +983,13 @@ class Agent(HookRegistry):
A state that already fits one window is passed straight to `system_one` (identical output).
+ The hooks wrap the inference that answers the state, which for a document needing several
+ windows is the one shared `predict_batch` over them: `on_predict_start` fires once, and
+ `ctx.states` holds the decoded window texts in scan order -- not the caller's `state`, which
+ was tokenized to produce them. A start hook that rewrites `ctx.states` therefore rewrites
+ windows, and one that calls `ctx.skip(...)` is answering the document rather than a window,
+ so its payload is returned without window attribution.
+
Args:
window: state tokens per window. Defaults to the per-question state budget
(`max_len - head_max_len - 8`) -- the most a window can hold for every question.
@@ -993,11 +1003,20 @@ class Agent(HookRegistry):
aggregate: "auto" (the per-type rules above) is the only mode for now.
batch_size: cap on windows per forward pass, to bound memory on very long states.
lang: per-language temperature selection, as in `system_one`.
+ hooks (HookArg): Per-call hooks, appended after any installed on the Agent.
+ See `laya.hooks`.
+ on_predict_start (PredictHookArg): A per-call start hook, as in `system_one`.
+ on_predict_end (PredictHookArg): A per-call end hook, as in `system_one`.
+ hooks_raise: Override the Agent's `hooks_raise` for this call.
+ hooks_timeout: Override the Agent's `hooks_timeout` for this call.
Returns a single result dict, the same shape as `system_one`, with `usage["windows"]` added.
"""
if aggregate != "auto":
raise ValueError("predict_long: only aggregate='auto' is supported")
+ hook_kwargs = {"hooks": hooks, "on_predict_start": on_predict_start,
+ "on_predict_end": on_predict_end, "hooks_raise": hooks_raise,
+ "hooks_timeout": hooks_timeout}
max_len = self.cfg.get("max_len", 512)
head_max_len = self.cfg.get("head_max_len", 192)
budget = window if (window and window > 0) else max(64, max_len - head_max_len - 8)
@@ -1006,7 +1025,7 @@ class Agent(HookRegistry):
add_special_tokens=False)["input_ids"]
# Fits in one window: identical to a plain call, no windowing overhead.
if len(state_ids) <= budget:
- return self.system_one(state, questions, lang=lang)
+ return self.system_one(state, questions, lang=lang, **hook_kwargs)
step = stride if (stride and stride > 0) else max(1, budget // 2)
windows, starts = [], []
@@ -1021,7 +1040,26 @@ class Agent(HookRegistry):
break
i += step
- results = self.predict_batch(windows, questions, batch_size=batch_size, lang=lang)
+ results = self.predict_batch(windows, questions, batch_size=batch_size, lang=lang,
+ **hook_kwargs)
+
+ if len(results) != len(windows):
+ # Only a hook can change the count: `ctx.skip(...)` runs before the split, so a hook
+ # answering the whole document has no way to know how many windows it will produce and
+ # returns one result. Aggregating over that payload would pick between answers that were
+ # never scored and name a deciding window that decided nothing, so pass the document
+ # answer through unattributed and say which windows were scored: none of them.
+ if len(results) != 1:
+ raise ValueError(
+ "predict_long: %d windows scored, %d results returned; a start hook that answers"
+ " a long state returns one result for the document" % (len(windows), len(results)))
+ warnings.warn("laya: predict_long: a hook answered the state before it was scanned, so "
+ "no window decided the result and none is reported", RuntimeWarning,
+ stacklevel=2)
+ document = dict(results[0])
+ document["usage"] = dict(document.get("usage") or {})
+ document["usage"]["windows"] = 0
+ return document
ids = list(questions.keys())
internal = {qid: self._to_internal(questions[qid]) for qid in ids}
diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py
index fa750e30..7dc6422e 100644
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@ -72,6 +72,7 @@ check_param("Router.__init__", Router.__init__, "revisions", None)
# --------------------------------------------------------------- predict surfaces
for label, fn in (("Agent.predict_batch", Agent.predict_batch),
("Agent.system_one", Agent.system_one),
+ ("Agent.predict_long", Agent.predict_long),
("Router.predict", Router.predict),
("ONNXAgent.system_one", ONNXAgent.system_one)):
check_param(label, fn, "hooks", None)
@@ -80,6 +81,13 @@ for label, fn in (("Agent.predict_batch", Agent.predict_batch),
check_param(label, fn, "hooks_raise", None)
check_param(label, fn, "hooks_timeout", None)
+# The scan sizes its windows from the checkpoint budget, so it takes `window`/`stride` instead
+# of the per-call token overrides the single-window entries take.
+check_param("Agent.predict_long", Agent.predict_long, "window", None)
+check_param("Agent.predict_long", Agent.predict_long, "stride", None)
+for param in ("max_len", "head_max_len"):
+ check("Agent.predict_long/%s not accepted" % param, param in sig(Agent.predict_long), False)
+
check_param("Agent.predict_batch", Agent.predict_batch, "batch_size", None)
# per-call token-budget overrides
diff --git a/tests/test_predict_long.py b/tests/test_predict_long.py
index e8d70688..d02eb3bc 100644
--- a/tests/test_predict_long.py
+++ b/tests/test_predict_long.py
@@ -2,11 +2,13 @@
Weight-free. The real forward path is stubbed (predict_batch returns canned per-window answers),
so this checks only predict_long's own logic: the fits-in-one-window short-circuit, the overlapping
-window split, and the per-type aggregation (noul = strongest window, choice/score = most-confident
-window). Numerical behaviour on real weights is exercised in tests/test_local_e2e.py.
+window split, the per-type aggregation (noul = strongest window, choice/score = most-confident
+window), and the per-call hook controls it forwards to whichever of those two calls runs. Numerical
+behaviour on real weights is exercised in tests/test_local_e2e.py.
"""
import os
import sys
+import warnings
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -46,14 +48,17 @@ def make_agent(batch_result_fn):
a.cfg = {"max_len": 100, "head_max_len": 20} # budget = max(64, 100-20-8) = 72
a.tok = _Tok()
a._to_internal = staticmethod(Agent._to_internal).__func__
- a._calls = {"system_one": 0, "batch_states": None}
+ a._calls = {"system_one": 0, "batch_states": None, "system_one_kwargs": None,
+ "batch_kwargs": None}
- def _system_one(state, questions, lang=None):
+ def _system_one(state, questions, lang=None, **controls):
a._calls["system_one"] += 1
+ a._calls["system_one_kwargs"] = controls
return {"model": "laya-rl-agent", "answers": {"_via": "system_one"}, "usage": {"input_tokens": 1}}
- def _predict_batch(states, questions, batch_size=None, lang=None):
+ def _predict_batch(states, questions, batch_size=None, lang=None, **controls):
a._calls["batch_states"] = list(states)
+ a._calls["batch_kwargs"] = controls
return batch_result_fn(list(states), questions)
a.system_one = _system_one
@@ -108,6 +113,66 @@ a = make_agent(canned)
check_raises("aggregate/rejects unknown mode", ValueError,
lambda: a.predict_long({"body": "y" * 300}, Q, aggregate="mean"))
+# 4. the per-call hook controls `predict`/`system_one`/`predict_batch` take reach the scan
+HOOK_KEYS = ("hooks", "on_predict_start", "on_predict_end", "hooks_raise", "hooks_timeout")
+LONG = {"body": "y" * 300}
+sentinel = object()
+a = make_agent(canned)
+res = a.predict_long(LONG, Q, hooks=[sentinel], hooks_raise=False, hooks_timeout=2.5)
+got = a._calls["batch_kwargs"]
+check("hooks/forwarded to the scan", sorted(got), sorted(HOOK_KEYS))
+check("hooks/the hook list reaches it", got.get("hooks"), [sentinel])
+check("hooks/hooks_raise reaches it", got.get("hooks_raise"), False)
+check("hooks/hooks_timeout reaches it", got.get("hooks_timeout"), 2.5)
+check("hooks/start/end default to None",
+ [got.get("on_predict_start"), got.get("on_predict_end")], [None, None])
+
+# 5. the same controls reach the fits-in-one-window path
+a = make_agent(lambda s, q: [])
+a.predict_long({"body": "x" * 50}, Q, hooks=[sentinel], hooks_timeout=1.25)
+got = a._calls["system_one_kwargs"]
+check("hooks/forwarded to system_one", sorted(got), sorted(HOOK_KEYS))
+check("hooks/the hook list reaches it there", got.get("hooks"), [sentinel])
+check("hooks/hooks_timeout reaches it there", got.get("hooks_timeout"), 1.25)
+
+# 6. what the scan forwards is the window texts, in scan order, not the caller's state object
+a = make_agent(canned)
+a.predict_long(LONG, Q)
+states = a._calls["batch_states"]
+check("states/several windows", len(states) > 1, True)
+check("states/decoded text, not the caller's dict",
+ all(isinstance(s, str) for s in states) and states[0] == "w0_71", True)
+check("states/they overlap", a._calls["batch_states"][1], "w36_107")
+
+# 7. forwarding the controls must not move a decision
+plain = make_agent(canned).predict_long(LONG, Q)
+with_hooks = make_agent(canned).predict_long(LONG, Q, hooks=[sentinel], hooks_timeout=9)
+check("hooks/forwarding is decision-neutral", with_hooks, plain)
+
+# 8. a start hook that answers the document is not attributed to a window that was never scored
+def one_document_answer(states, q):
+ return [{"answers": {"dept": {"type": "choice", "choice": "a", "answer_confidence": 0.7},
+ "flag": {"type": "noul", "noul": 0.2, "answer_confidence": 0.8}},
+ "usage": {"input_tokens": 3, "output_tokens": 0}}]
+
+
+a = make_agent(one_document_answer)
+with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ res = a.predict_long(LONG, Q)
+nwin = len(a._calls["batch_states"])
+check("skip/the state was split into several windows", nwin > 1, True)
+check("skip/no window was scored", res["usage"]["windows"], 0)
+check("skip/the payload tokens are kept", res["usage"]["input_tokens"], 3)
+check("skip/no deciding window is claimed",
+ sorted(k for v in res["answers"].values() for k in v if k == "window"), [])
+check("skip/the hook's answer is returned", res["answers"]["dept"]["choice"], "a")
+check("skip/the caller is told", [w.category.__name__ for w in caught], ["RuntimeWarning"])
+
+# 9. a payload that matches neither the document nor the windows is an error, not a guess
+a = make_agent(lambda s, q: [{"answers": {}, "usage": {}}] * 2)
+check_raises("skip/rejects a count matching nothing", ValueError, lambda: a.predict_long(LONG, Q))
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL " + f)