Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(router): predict_long -- route, then scan the whole state in windows
Aashish9810daa57afe
diff --git a/README.md b/README.mdindex 54c20cf9..45215ba7 100644--- a/README.md+++ b/README.md@@ -581,6 +581,13 @@ A smaller `window` isolates a short deciding span better (it becomes a larger frwindow); the default (`max_len - head_max_len`) favors context and throughput. Output shape matches`predict`, with `usage["windows"]` added.+The same scan is available from the Router, which routes first and then windows the checkpoint it+chose — same hints (`model=`, `task=`, `lang=`), same hooks, same `routing` key:++```python+result = router.predict_long(state, questions, model="multilingual")+```+The returned probability is the deciding window's, **not a calibrated number for the wholedocument** — a `noul` max drifts up with the window count even with no signal, and `choice` can landon a confidently-neutral window when nothing is decisive. Each answer carries `answer["window"]`diff --git a/laya/router.py b/laya/router.pyindex 7dd999eb..2b90cb6f 100644--- a/laya/router.py+++ b/laya/router.py@@ -174,6 +174,40 @@ def _english_from_code(value: Any) -> Optional[bool]:return primary in _ENGLISH_SUBTAGS+class _ScanLong:+ """Last start hook of `Router.predict_long`: score the routed state with the agent's+ `predict_long` rather than leave `predict` to call `system_one` on one window of it.++ It is appended after every other start hook, so a caller's hook has already had its say: one+ that answered (`ctx.skip(...)`) or rewrote the state/questions wins, and only what is left is+ scanned. The `lang` rule is `predict`'s -- an explicit `lang=`, else the language routing+ detected -- repeated here because `predict` computes it locally for its own call.+ """++ def __init__(self, window, stride, aggregate, batch_size, lang):+ self.window = window+ self.stride = stride+ self.aggregate = aggregate+ self.batch_size = batch_size+ self.lang = lang++ def on_predict_start(self, ctx):+ if ctx.results is not None:+ return+ if not hasattr(ctx.agent, "predict_long"):+ raise TypeError(+ "%s has no predict_long, so a state longer than its window cannot be scanned; "+ "the PyTorch Agent implements it, and an ONNX or hand-attached agent needs it too"+ % type(ctx.agent).__name__+ )+ lang = self.lang+ if lang is None:+ lang = (ctx.decision.get("detection") or {}).get("language")+ ctx.results = [ctx.agent.predict_long(+ ctx.states[0], ctx.questions, window=self.window, stride=self.stride,+ aggregate=self.aggregate, batch_size=self.batch_size, lang=lang)]++class Router(HookRegistry):"""Lazily loads Laya checkpoints and sends each request to the right one.@@ -636,6 +670,50 @@ class Router(HookRegistry):raisereturn ctx.results[0]+ def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Any],+ model: Optional[str] = None, task: Optional[str] = None,+ lang: Optional[str] = None, lang_guess: Optional[Any] = None,+ window: Optional[int] = None, stride: Optional[int] = None,+ aggregate: str = "auto", batch_size: Optional[int] = None,+ hooks=None, on_predict_start=None, on_predict_end=None,+ hooks_raise: Optional[bool] = None, hooks_timeout: Optional[float] = None,+ ) -> Dict[str, Any]:+ """Route, then scan every window of the state instead of only its first one.++ `predict` scores a state from a single window: anything past `max_len` is cut off (the+ first window, or for a conversation list the last) and never reaches the model. This+ routes exactly as `predict` does -- the same `model`/`task`/`lang` hints, the same+ router-level hooks, the same `routing` key and `usage` -- and scores the routed state with+ that agent's `predict_long`, which splits it into overlapping windows and aggregates per+ question. The aggregation rules are `laya.agent.Agent.predict_long`'s: `noul` takes the+ strongest window, `choice`/`score` the most confident one.++ Args:+ window: state tokens per window. Defaults to the routed checkpoint's budget+ (`max_len - head_max_len - 8`); a smaller window isolates a localized span.+ stride: token step between windows; defaults to `window // 2` (50% overlap).+ aggregate: "auto" (the per-type rules above) is the only mode.+ batch_size: cap on windows per forward pass, to bound memory on very long states.+ hooks, on_predict_start, on_predict_end, hooks_raise, hooks_timeout: wrap the whole+ route+scan exactly as they wrap `predict`. `max_len` / `head_max_len` are not+ accepted: a window is sized by `window` or the checkpoint budget, and+ overriding the single-window truncation is what `predict_long` is for.++ Returns:+ The usual `predict` payload, with `usage["windows"]` counting the windows scored.++ Raises:+ TypeError: the routed agent has no `predict_long` (an ONNX agent, or one attached by+ hand), so there is nothing to scan with. Raised under the router's+ `hooks_raise` policy, which defaults to raising.+ """+ per_call = normalise_hooks(hooks, on_predict_start, on_predict_end)+ per_call.append(_ScanLong(window=window, stride=stride, aggregate=aggregate,+ batch_size=batch_size, lang=lang))+ return self.predict(state, questions, model=model, task=task, lang=lang,+ lang_guess=lang_guess, hooks=per_call,+ hooks_raise=hooks_raise, hooks_timeout=hooks_timeout)+def decide(self, state: Union[str, dict, list], schema: Any = None, *,questions: Optional[Dict[str, Any]] = None, return_details: bool = False,**predict_kwargs) -> Any:diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.pyindex fa750e30..fa3a2df7 100644--- a/tests/test_hooks_api.py+++ b/tests/test_hooks_api.py@@ -73,6 +73,7 @@ check_param("Router.__init__", Router.__init__, "revisions", None)for label, fn in (("Agent.predict_batch", Agent.predict_batch),("Agent.system_one", Agent.system_one),("Router.predict", Router.predict),+ ("Router.predict_long", Router.predict_long),("ONNXAgent.system_one", ONNXAgent.system_one)):check_param(label, fn, "hooks", None)check_param(label, fn, "on_predict_start", None)diff --git a/tests/test_predict_long.py b/tests/test_predict_long.pyindex e8d70688..88c34710 100644--- a/tests/test_predict_long.py+++ b/tests/test_predict_long.py@@ -108,6 +108,99 @@ a = make_agent(canned)check_raises("aggregate/rejects unknown mode", ValueError,lambda: a.predict_long({"body": "y" * 300}, Q, aggregate="mean"))+# ------------------------------------------------------------------- Router.predict_long+# The Router is the entry point the README leads with, and it had no windowed scan: a state past+# max_len was answered from its first window however it was routed. Weight-free, like the rest of+# this file -- the routed agent is a stub, attached so no checkpoint is ever built.+from laya.router import Router # noqa: E402+++class _LongStub:+ def __init__(self):+ self.calls = []++ def predict_long(self, state, questions, window=None, stride=None, aggregate="auto",+ batch_size=None, lang=None):+ self.calls.append({"state": state, "questions": questions, "window": window,+ "stride": stride, "aggregate": aggregate,+ "batch_size": batch_size, "lang": lang})+ return {"model": "stub", "answers": {"scanned": {"noul": 0.5}},+ "usage": {"windows": 3, "input_tokens": 30}}+++def _router(stub, hooks=None):+ r = Router(hooks=hooks) if hooks else Router()+ r.attach("english", stub)+ return r+++# 4. routes first, then scans on the routed agent, with the caller's options forwarded+stub = _LongStub()+out = _router(stub).predict_long({"body": "y" * 300}, Q, model="english",+ window=64, stride=32, batch_size=8, lang="de")+check("router/routing key attached", out["routing"]["model"], "english")+check("router/answers come from the scan", out["answers"], {"scanned": {"noul": 0.5}})+check("router/window forwarded", stub.calls[0]["window"], 64)+check("router/stride forwarded", stub.calls[0]["stride"], 32)+check("router/batch_size forwarded", stub.calls[0]["batch_size"], 8)+check("router/explicit lang forwarded", stub.calls[0]["lang"], "de")+check("router/usage carried through", out["usage"]["windows"], 3)++# 5. an installed start hook runs before the scan and may answer instead of it+class _Skipper:+ def __init__(self):+ self.started = 0++ def on_predict_start(self, ctx):+ self.started += 1+ ctx.skip([{"model": "hook", "answers": {"cached": {"noul": 0.9}}, "usage": {}}])+++skipper = _Skipper()+skipped_stub = _LongStub()+out = _router(skipped_stub, hooks=[skipper]).predict_long(+ {"body": "y" * 300}, Q, model="english")+check("router/installed start hook ran", skipper.started, 1)+check("router/hook answer wins over the scan", skipped_stub.calls, [])+check("router/skipped answer still routed", out["routing"]["model"], "english")++# 6. a start hook that rewrites the state: the scan reads what it left behind+class _Rewriter:+ def on_predict_start(self, ctx):+ ctx.states[0] = "rewritten by the start hook"+++rewritten_stub = _LongStub()+_router(rewritten_stub, hooks=[_Rewriter()]).predict_long(+ {"body": "y" * 300}, Q, model="english")+check("router/scan uses the rewritten state", rewritten_stub.calls[0]["state"],+ "rewritten by the start hook")++# 7. a per-call end hook sees the scanned result+ended = {}+++class _Recorder:+ def on_predict_end(self, ctx):+ ended["answers"] = ctx.results[0]["answers"]+++recorder_stub = _LongStub()+_router(recorder_stub).predict_long({"body": "y" * 300}, Q, model="english",+ on_predict_end=_Recorder().on_predict_end)+check("router/per-call end hook sees the scan", ended["answers"], {"scanned": {"noul": 0.5}})++# 8. an agent with no predict_long is a named caller error, not a bare AttributeError+class _NoScan:+ def system_one(self, state, questions):+ return {"model": "noscan", "answers": {}, "usage": {}}+++r_noscan = Router()+r_noscan.attach("english", _NoScan())+check_raises("router/agent without predict_long", TypeError,+ lambda: r_noscan.predict_long({"body": "y" * 300}, Q, model="english"))+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL:print(" FAIL " + f)