Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(router): predict_long -- route, then scan the whole state in windows

Aashish9810daa57afe
diff --git a/README.md b/README.md
index 54c20cf9..45215ba7 100644
--- a/README.md
+++ b/README.md
@@ -581,6 +581,13 @@ A smaller `window` isolates a short deciding span better (it becomes a larger fr
window); the default (`max_len - head_max_len`) favors context and throughput. Output shape matches
`predict`, with `usage["windows"]` added.
+The same scan is available from the Router, which routes first and then windows the checkpoint it
+chose — same hints (`model=`, `task=`, `lang=`), same hooks, same `routing` key:
+
+```python
+result = router.predict_long(state, questions, model="multilingual")
+```
+
The returned probability is the deciding window's, **not a calibrated number for the whole
document** — a `noul` max drifts up with the window count even with no signal, and `choice` can land
on a confidently-neutral window when nothing is decisive. Each answer carries `answer["window"]`
diff --git a/laya/router.py b/laya/router.py
index 7dd999eb..2b90cb6f 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -174,6 +174,40 @@ def _english_from_code(value: Any) -> Optional[bool]:
return primary in _ENGLISH_SUBTAGS
+class _ScanLong:
+ """Last start hook of `Router.predict_long`: score the routed state with the agent's
+ `predict_long` rather than leave `predict` to call `system_one` on one window of it.
+
+ It is appended after every other start hook, so a caller's hook has already had its say: one
+ that answered (`ctx.skip(...)`) or rewrote the state/questions wins, and only what is left is
+ scanned. The `lang` rule is `predict`'s -- an explicit `lang=`, else the language routing
+ detected -- repeated here because `predict` computes it locally for its own call.
+ """
+
+ def __init__(self, window, stride, aggregate, batch_size, lang):
+ self.window = window
+ self.stride = stride
+ self.aggregate = aggregate
+ self.batch_size = batch_size
+ self.lang = lang
+
+ def on_predict_start(self, ctx):
+ if ctx.results is not None:
+ return
+ if not hasattr(ctx.agent, "predict_long"):
+ raise TypeError(
+ "%s has no predict_long, so a state longer than its window cannot be scanned; "
+ "the PyTorch Agent implements it, and an ONNX or hand-attached agent needs it too"
+ % type(ctx.agent).__name__
+ )
+ lang = self.lang
+ if lang is None:
+ lang = (ctx.decision.get("detection") or {}).get("language")
+ ctx.results = [ctx.agent.predict_long(
+ ctx.states[0], ctx.questions, window=self.window, stride=self.stride,
+ aggregate=self.aggregate, batch_size=self.batch_size, lang=lang)]
+
+
class Router(HookRegistry):
"""Lazily loads Laya checkpoints and sends each request to the right one.
@@ -636,6 +670,50 @@ class Router(HookRegistry):
raise
return ctx.results[0]
+ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Any],
+ model: Optional[str] = None, task: Optional[str] = None,
+ lang: Optional[str] = None, lang_guess: Optional[Any] = None,
+ window: Optional[int] = None, stride: Optional[int] = None,
+ aggregate: str = "auto", batch_size: Optional[int] = None,
+ hooks=None, on_predict_start=None, on_predict_end=None,
+ hooks_raise: Optional[bool] = None, hooks_timeout: Optional[float] = None,
+ ) -> Dict[str, Any]:
+ """Route, then scan every window of the state instead of only its first one.
+
+ `predict` scores a state from a single window: anything past `max_len` is cut off (the
+ first window, or for a conversation list the last) and never reaches the model. This
+ routes exactly as `predict` does -- the same `model`/`task`/`lang` hints, the same
+ router-level hooks, the same `routing` key and `usage` -- and scores the routed state with
+ that agent's `predict_long`, which splits it into overlapping windows and aggregates per
+ question. The aggregation rules are `laya.agent.Agent.predict_long`'s: `noul` takes the
+ strongest window, `choice`/`score` the most confident one.
+
+ Args:
+ window: state tokens per window. Defaults to the routed checkpoint's budget
+ (`max_len - head_max_len - 8`); a smaller window isolates a localized span.
+ stride: token step between windows; defaults to `window // 2` (50% overlap).
+ aggregate: "auto" (the per-type rules above) is the only mode.
+ batch_size: cap on windows per forward pass, to bound memory on very long states.
+ hooks, on_predict_start, on_predict_end, hooks_raise, hooks_timeout: wrap the whole
+ route+scan exactly as they wrap `predict`. `max_len` / `head_max_len` are not
+ accepted: a window is sized by `window` or the checkpoint budget, and
+ overriding the single-window truncation is what `predict_long` is for.
+
+ Returns:
+ The usual `predict` payload, with `usage["windows"]` counting the windows scored.
+
+ Raises:
+ TypeError: the routed agent has no `predict_long` (an ONNX agent, or one attached by
+ hand), so there is nothing to scan with. Raised under the router's
+ `hooks_raise` policy, which defaults to raising.
+ """
+ per_call = normalise_hooks(hooks, on_predict_start, on_predict_end)
+ per_call.append(_ScanLong(window=window, stride=stride, aggregate=aggregate,
+ batch_size=batch_size, lang=lang))
+ return self.predict(state, questions, model=model, task=task, lang=lang,
+ lang_guess=lang_guess, hooks=per_call,
+ hooks_raise=hooks_raise, hooks_timeout=hooks_timeout)
+
def decide(self, state: Union[str, dict, list], schema: Any = None, *,
questions: Optional[Dict[str, Any]] = None, return_details: bool = False,
**predict_kwargs) -> Any:
diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py
index fa750e30..fa3a2df7 100644
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@ -73,6 +73,7 @@ check_param("Router.__init__", Router.__init__, "revisions", None)
for label, fn in (("Agent.predict_batch", Agent.predict_batch),
("Agent.system_one", Agent.system_one),
("Router.predict", Router.predict),
+ ("Router.predict_long", Router.predict_long),
("ONNXAgent.system_one", ONNXAgent.system_one)):
check_param(label, fn, "hooks", None)
check_param(label, fn, "on_predict_start", None)
diff --git a/tests/test_predict_long.py b/tests/test_predict_long.py
index e8d70688..88c34710 100644
--- a/tests/test_predict_long.py
+++ b/tests/test_predict_long.py
@@ -108,6 +108,99 @@ a = make_agent(canned)
check_raises("aggregate/rejects unknown mode", ValueError,
lambda: a.predict_long({"body": "y" * 300}, Q, aggregate="mean"))
+# ------------------------------------------------------------------- Router.predict_long
+# The Router is the entry point the README leads with, and it had no windowed scan: a state past
+# max_len was answered from its first window however it was routed. Weight-free, like the rest of
+# this file -- the routed agent is a stub, attached so no checkpoint is ever built.
+from laya.router import Router # noqa: E402
+
+
+class _LongStub:
+ def __init__(self):
+ self.calls = []
+
+ def predict_long(self, state, questions, window=None, stride=None, aggregate="auto",
+ batch_size=None, lang=None):
+ self.calls.append({"state": state, "questions": questions, "window": window,
+ "stride": stride, "aggregate": aggregate,
+ "batch_size": batch_size, "lang": lang})
+ return {"model": "stub", "answers": {"scanned": {"noul": 0.5}},
+ "usage": {"windows": 3, "input_tokens": 30}}
+
+
+def _router(stub, hooks=None):
+ r = Router(hooks=hooks) if hooks else Router()
+ r.attach("english", stub)
+ return r
+
+
+# 4. routes first, then scans on the routed agent, with the caller's options forwarded
+stub = _LongStub()
+out = _router(stub).predict_long({"body": "y" * 300}, Q, model="english",
+ window=64, stride=32, batch_size=8, lang="de")
+check("router/routing key attached", out["routing"]["model"], "english")
+check("router/answers come from the scan", out["answers"], {"scanned": {"noul": 0.5}})
+check("router/window forwarded", stub.calls[0]["window"], 64)
+check("router/stride forwarded", stub.calls[0]["stride"], 32)
+check("router/batch_size forwarded", stub.calls[0]["batch_size"], 8)
+check("router/explicit lang forwarded", stub.calls[0]["lang"], "de")
+check("router/usage carried through", out["usage"]["windows"], 3)
+
+# 5. an installed start hook runs before the scan and may answer instead of it
+class _Skipper:
+ def __init__(self):
+ self.started = 0
+
+ def on_predict_start(self, ctx):
+ self.started += 1
+ ctx.skip([{"model": "hook", "answers": {"cached": {"noul": 0.9}}, "usage": {}}])
+
+
+skipper = _Skipper()
+skipped_stub = _LongStub()
+out = _router(skipped_stub, hooks=[skipper]).predict_long(
+ {"body": "y" * 300}, Q, model="english")
+check("router/installed start hook ran", skipper.started, 1)
+check("router/hook answer wins over the scan", skipped_stub.calls, [])
+check("router/skipped answer still routed", out["routing"]["model"], "english")
+
+# 6. a start hook that rewrites the state: the scan reads what it left behind
+class _Rewriter:
+ def on_predict_start(self, ctx):
+ ctx.states[0] = "rewritten by the start hook"
+
+
+rewritten_stub = _LongStub()
+_router(rewritten_stub, hooks=[_Rewriter()]).predict_long(
+ {"body": "y" * 300}, Q, model="english")
+check("router/scan uses the rewritten state", rewritten_stub.calls[0]["state"],
+ "rewritten by the start hook")
+
+# 7. a per-call end hook sees the scanned result
+ended = {}
+
+
+class _Recorder:
+ def on_predict_end(self, ctx):
+ ended["answers"] = ctx.results[0]["answers"]
+
+
+recorder_stub = _LongStub()
+_router(recorder_stub).predict_long({"body": "y" * 300}, Q, model="english",
+ on_predict_end=_Recorder().on_predict_end)
+check("router/per-call end hook sees the scan", ended["answers"], {"scanned": {"noul": 0.5}})
+
+# 8. an agent with no predict_long is a named caller error, not a bare AttributeError
+class _NoScan:
+ def system_one(self, state, questions):
+ return {"model": "noscan", "answers": {}, "usage": {}}
+
+
+r_noscan = Router()
+r_noscan.attach("english", _NoScan())
+check_raises("router/agent without predict_long", TypeError,
+ lambda: r_noscan.predict_long({"body": "y" * 300}, Q, model="english"))
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL " + f)