Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #577 from aashish254/feat-predict-long-hooks

Nandakishord6f1f3e3a316
diff --cc laya/agent.py
index 873be131,66e7e60c..3d34fa0f
--- a/laya/agent.py
+++ b/laya/agent.py
@@@ -30,10 -28,9 +30,10 @@@ from .common import
serialize_state,
temp_bucket,
)
+from .confidence import check_min_confidence, flag_low_confidence
from .hooks import (
- HookRegistry, PredictContext, aggregate_usage, compose_hooks, dispatch, normalise_hooks,
- validate_timeout,
+ HookRegistry, PredictContext, _as_sequence, aggregate_usage, compose_hooks, dispatch,
+ normalise_hooks, validate_timeout,
)
from .revisions import resolve_revision, snapshot_revision, verify_digests
diff --cc tests/test_hooks_api.py
index ffaeac02,7dc6422e..65f1054c
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@@ -76,8 -72,8 +76,9 @@@ check_param("Router.__init__", Router._
# --------------------------------------------------------------- predict surfaces
for label, fn in (("Agent.predict_batch", Agent.predict_batch),
("Agent.system_one", Agent.system_one),
+ ("Agent.predict_long", Agent.predict_long),
("Router.predict", Router.predict),
+ ("Router.predict_long", Router.predict_long),
("ONNXAgent.system_one", ONNXAgent.system_one)):
check_param(label, fn, "hooks", None)
check_param(label, fn, "on_predict_start", None)
@@@ -85,10 -81,14 +86,17 @@@
check_param(label, fn, "hooks_raise", None)
check_param(label, fn, "hooks_timeout", None)
+ # The scan sizes its windows from the checkpoint budget, so it takes `window`/`stride` instead
+ # of the per-call token overrides the single-window entries take.
+ check_param("Agent.predict_long", Agent.predict_long, "window", None)
+ check_param("Agent.predict_long", Agent.predict_long, "stride", None)
+ for param in ("max_len", "head_max_len"):
+ check("Agent.predict_long/%s not accepted" % param, param in sig(Agent.predict_long), False)
+
check_param("Agent.predict_batch", Agent.predict_batch, "batch_size", None)
+check_param("Agent.predict_batch", Agent.predict_batch, "sort_by_length", False)
+check_param("Router.predict_batch", Router.predict_batch, "batch_size", None)
+check_param("Router.predict_batch", Router.predict_batch, "sort_by_length", False)
# per-call token-budget overrides
for label, fn in (("Agent.predict_batch", Agent.predict_batch),
diff --cc tests/test_predict_long.py
index 88c34710,8325d2c2..67d54d69
--- a/tests/test_predict_long.py
+++ b/tests/test_predict_long.py
@@@ -108,99 -175,234 +175,327 @@@ a = make_agent(canned
check_raises("aggregate/rejects unknown mode", ValueError,
lambda: a.predict_long({"body": "y" * 300}, Q, aggregate="mean"))
+# ------------------------------------------------------------------- Router.predict_long
+# The Router is the entry point the README leads with, and it had no windowed scan: a state past
+# max_len was answered from its first window however it was routed. Weight-free, like the rest of
+# this file -- the routed agent is a stub, attached so no checkpoint is ever built.
+from laya.router import Router # noqa: E402
+
+
+class _LongStub:
+ def __init__(self):
+ self.calls = []
+
+ def predict_long(self, state, questions, window=None, stride=None, aggregate="auto",
+ batch_size=None, lang=None):
+ self.calls.append({"state": state, "questions": questions, "window": window,
+ "stride": stride, "aggregate": aggregate,
+ "batch_size": batch_size, "lang": lang})
+ return {"model": "stub", "answers": {"scanned": {"noul": 0.5}},
+ "usage": {"windows": 3, "input_tokens": 30}}
+
+
+def _router(stub, hooks=None):
+ r = Router(hooks=hooks) if hooks else Router()
+ r.attach("english", stub)
+ return r
+
+
+# 4. routes first, then scans on the routed agent, with the caller's options forwarded
+stub = _LongStub()
+out = _router(stub).predict_long({"body": "y" * 300}, Q, model="english",
+ window=64, stride=32, batch_size=8, lang="de")
+check("router/routing key attached", out["routing"]["model"], "english")
+check("router/answers come from the scan", out["answers"], {"scanned": {"noul": 0.5}})
+check("router/window forwarded", stub.calls[0]["window"], 64)
+check("router/stride forwarded", stub.calls[0]["stride"], 32)
+check("router/batch_size forwarded", stub.calls[0]["batch_size"], 8)
+check("router/explicit lang forwarded", stub.calls[0]["lang"], "de")
+check("router/usage carried through", out["usage"]["windows"], 3)
+
+# 5. an installed start hook runs before the scan and may answer instead of it
+class _Skipper:
+ def __init__(self):
+ self.started = 0
+
+ def on_predict_start(self, ctx):
+ self.started += 1
+ ctx.skip([{"model": "hook", "answers": {"cached": {"noul": 0.9}}, "usage": {}}])
+
+
+skipper = _Skipper()
+skipped_stub = _LongStub()
+out = _router(skipped_stub, hooks=[skipper]).predict_long(
+ {"body": "y" * 300}, Q, model="english")
+check("router/installed start hook ran", skipper.started, 1)
+check("router/hook answer wins over the scan", skipped_stub.calls, [])
+check("router/skipped answer still routed", out["routing"]["model"], "english")
+
+# 6. a start hook that rewrites the state: the scan reads what it left behind
+class _Rewriter:
+ def on_predict_start(self, ctx):
+ ctx.states[0] = "rewritten by the start hook"
+
+
+rewritten_stub = _LongStub()
+_router(rewritten_stub, hooks=[_Rewriter()]).predict_long(
+ {"body": "y" * 300}, Q, model="english")
+check("router/scan uses the rewritten state", rewritten_stub.calls[0]["state"],
+ "rewritten by the start hook")
+
+# 7. a per-call end hook sees the scanned result
+ended = {}
+
+
+class _Recorder:
+ def on_predict_end(self, ctx):
+ ended["answers"] = ctx.results[0]["answers"]
+
+
+recorder_stub = _LongStub()
+_router(recorder_stub).predict_long({"body": "y" * 300}, Q, model="english",
+ on_predict_end=_Recorder().on_predict_end)
+check("router/per-call end hook sees the scan", ended["answers"], {"scanned": {"noul": 0.5}})
+
+# 8. an agent with no predict_long is a named caller error, not a bare AttributeError
+class _NoScan:
+ def system_one(self, state, questions):
+ return {"model": "noscan", "answers": {}, "usage": {}}
+
+
+r_noscan = Router()
+r_noscan.attach("english", _NoScan())
+check_raises("router/agent without predict_long", TypeError,
+ lambda: r_noscan.predict_long({"body": "y" * 300}, Q, model="english"))
+
+ # 4. the per-call hook controls `predict`/`system_one`/`predict_batch` take reach the scan
+ HOOK_KEYS = ("hooks", "on_predict_start", "on_predict_end", "hooks_raise", "hooks_timeout")
+ LONG = {"body": "y" * 300}
+ sentinel = object()
+
+
+ def sentinel_start(ctx):
+ pass
+
+
+ a = make_agent(canned)
+ res = a.predict_long(LONG, Q, hooks=[sentinel], on_predict_start=sentinel_start,
+ hooks_raise=False, hooks_timeout=2.5)
+ got = a._calls["batch_kwargs"]
+ check("hooks/forwarded to the scan", sorted(got), sorted(HOOK_KEYS))
+ check("hooks/the hook list reaches it", got.get("hooks"), [sentinel])
+ check("hooks/hooks_raise reaches it", got.get("hooks_raise"), False)
+ check("hooks/hooks_timeout reaches it", got.get("hooks_timeout"), 2.5)
+ started = got.get("on_predict_start")
+ started = started if isinstance(started, list) else ([] if started is None else [started])
+ check("hooks/the caller's start hook still runs first",
+ bool(started) and started[0] is sentinel_start, True)
+ check("hooks/end stays absent when the caller sets none", got.get("on_predict_end"), None)
+
+ # 5. the same controls reach the fits-in-one-window path
+ a = make_agent(lambda s, q: [])
+ a.predict_long({"body": "x" * 50}, Q, hooks=[sentinel], hooks_timeout=1.25)
+ got = a._calls["system_one_kwargs"]
+ check("hooks/forwarded to system_one", sorted(got), sorted(HOOK_KEYS))
+ check("hooks/the hook list reaches it there", got.get("hooks"), [sentinel])
+ check("hooks/hooks_timeout reaches it there", got.get("hooks_timeout"), 1.25)
+
+ # 6. what the scan forwards is the window texts, in scan order, not the caller's state object
+ a = make_agent(canned)
+ a.predict_long(LONG, Q)
+ states = a._calls["batch_states"]
+ check("states/several windows", len(states) > 1, True)
+ check("states/decoded text, not the caller's dict",
+ all(isinstance(s, str) for s in states) and states[0] == "w0_71", True)
+ check("states/they overlap", a._calls["batch_states"][1], "w36_107")
+
+ # 7. forwarding the controls must not move a decision
+ plain = make_agent(canned).predict_long(LONG, Q)
+ with_hooks = make_agent(canned).predict_long(LONG, Q, hooks=[sentinel], hooks_timeout=9)
+ check("hooks/forwarding is decision-neutral", with_hooks, plain)
+
+ # 8. a start hook that answers the document, on the engine's own evidence
+ DOC = {"model": "laya-rl-agent",
+ "answers": {"dept": {"type": "choice", "choice": "b", "answer_confidence": 0.9},
+ "flag": {"type": "noul", "noul": 0.2, "answer_confidence": 0.8}},
+ "usage": {"input_tokens": 3, "output_tokens": 0}}
+
+ # The instrument for sections 8-8e: a predict_long call is captured rather than let propagate, so
+ # a case that ends in the wrong shape -- an answer that should have come back raising, or a
+ # rejection that should not have happened at all -- fails a named check instead of aborting the run
+ # and hiding every check after it.
+ def _attempt(fn):
+ """Call `fn`, returning (value, exception)."""
+ try:
+ return fn(), None
+ except BaseException as exc: # noqa: BLE001
+ return None, exc
+
+
+ def _kind(exc):
+ return exc.__class__.__name__ if exc else None
+
+
+ # The control first: how many windows this state really splits into, measured from the forward
+ # passes it costs rather than from the key under test. Two questions, one row each per window.
+ scan = make_real_agent()
+ base, base_exc = _attempt(lambda: scan.predict_long(LONG, Q))
+ check("scan/an unhooked scan returns", _kind(base_exc), None)
+ nwin = sum(scan._forward_calls) // 2
+ check("scan/more than one window", nwin > 1, True)
+ check("scan/the scan reports the windows it read",
+ ((base or {}).get("usage") or {}).get("windows", "<absent>"), nwin)
+ check("scan/the deciding window is the last",
+ ((base or {}).get("answers") or {}).get("dept", {}).get("window", {}).get("index"), nwin - 1)
+
+
+ def _narrow(keep):
+ """A start hook that truncates the window list and lets inference run on what is left."""
+ def hook(ctx):
+ ctx.states = list(ctx.states)[:keep]
+ return hook
+
+
+ a = make_real_agent()
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ res, skip_exc = _attempt(
+ lambda: a.predict_long(LONG, Q, on_predict_start=lambda ctx: ctx.skip([DOC])))
+ check("skip/one payload for the document is never refused as a count mismatch", _kind(skip_exc), None)
+ check("skip/no forward pass ran", a._forward_calls, [])
+ usage, answers = (res or {}).get("usage", {}), (res or {}).get("answers", {})
+ check("skip/no window was scored", usage.get("windows", "<absent>"), 0)
+ check("skip/the payload tokens are kept", usage.get("input_tokens", "<absent>"), 3)
+ check("skip/the hook's answer is returned", answers.get("dept", {}).get("choice"), "b")
+ check("skip/no deciding window is claimed",
+ sorted(k for v in answers.values() for k in v if k == "window"), [])
+ check("skip/the caller is told", [w.category.__name__ for w in caught], ["RuntimeWarning"])
+
+ # 8b. a start hook that replaces the scan is the middle row: inference ran on the hook's states, so
+ # the answers aggregate, the count is theirs, and no span of the caller's document is named -- the
+ # offsets computed above describe windows that were not scored.
+ a = make_real_agent()
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ narrow, narrow_exc = _attempt(lambda: a.predict_long(LONG, Q, on_predict_start=_narrow(1)))
+ check("narrow/one state left behind is a scan, not a rejected count", _kind(narrow_exc), None)
+ # Why the forward sits next to the count: it is the proof that tokens were spent, which is exactly
+ # what made this case read as a hook answer before.
+ check("narrow/inference ran on the state that was left", a._forward_calls, [2])
+ check("narrow/the count is the state scored, not the split made",
+ ((narrow or {}).get("usage") or {}).get("windows", "<absent>"), 1)
+ check("narrow/the tokens that forward booked are kept",
+ ((narrow or {}).get("usage") or {}).get("input_tokens", 0) > 0, True)
+ check("narrow/no span of the caller's document is named",
+ sorted(k for v in (narrow or {}).get("answers", {}).values() for k in v if k == "window"), [])
+ check("narrow/the answer is still aggregated",
+ (narrow or {}).get("answers", {}).get("dept", {}).get("type"), "choice")
+ check("narrow/a rewrite is not warned about as a hook answer",
+ [w.category.__name__ for w in caught], [])
+ a = make_real_agent()
+ two, exc = _attempt(lambda: a.predict_long(LONG, Q, on_predict_start=_narrow(2)))
+ check("narrow/two states left behind are counted as two",
+ ((two or {}).get("usage") or {}).get("windows", "<absent>"), 2)
+ check("narrow/and two states is what the forward saw", a._forward_calls, [4])
+ # the other direction, and in place: `predict_long` hands the hook a copy of its split, so the
+ # state the hook invented is scored and counted without moving the windows computed above.
+ a = make_real_agent()
+ grown, exc = _attempt(
+ lambda: a.predict_long(LONG, Q, on_predict_start=lambda ctx: ctx.states.append("invented")))
+ check("grow/scoring one invented state is not an error", _kind(exc), None)
+ check("grow/the hook's added state is scored",
+ ((grown or {}).get("usage") or {}).get("windows", "<absent>"), nwin + 1)
+ check("grow/the caller's split is the one that was extended", a._forward_calls, [2 * (nwin + 1)])
+ check("grow/and no span is named",
+ sorted(k for v in (grown or {}).get("answers", {}).values() for k in v if k == "window"), [])
+
+ # 8c. an unchanged scan keeps the attribution; any rewrite drops it, even one of the same length
+ a = make_real_agent()
+ noop, exc = _attempt(
+ lambda: a.predict_long(LONG, Q, on_predict_start=lambda ctx: ctx.states.extend(())))
+ check("rewrite/a no-op on the list changes nothing", _kind(exc), None)
+ check("rewrite/a no-op keeps the count", ((noop or {}).get("usage") or {}).get("windows", "<absent>"), nwin)
+ check("rewrite/a no-op keeps the deciding window",
+ ((noop or {}).get("answers") or {}).get("dept", {}).get("window", {}).get("index"), nwin - 1)
+ a = make_real_agent()
+ upper, exc = _attempt(lambda: a.predict_long(
+ LONG, Q, on_predict_start=lambda ctx: setattr(ctx, "states", [s.upper() for s in ctx.states])))
+ check("rewrite/the hook's text is what was scored", (a._encoded or [""])[0].isupper(), True)
+ check("rewrite/the same length is still the same count",
+ ((upper or {}).get("usage") or {}).get("windows", "<absent>"), nwin)
+ check("rewrite/a text rewrite names no span",
+ sorted(k for v in (upper or {}).get("answers", {}).values() for k in v if k == "window"), [])
+
+ # 8e. a hook that leaves nothing scores nothing: 0 windows and no answers, not a max() over []
+ a = make_real_agent()
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ empty, exc = _attempt(
+ lambda: a.predict_long(LONG, Q, on_predict_start=lambda ctx: setattr(ctx, "states", [])))
+ check("empty/nothing scored returns instead of raising", _kind(exc), None)
+ check("empty/no answer is invented", (empty or {"answers": "?"}).get("answers", "?"), {})
+ check("empty/the count says so", ((empty or {}).get("usage") or {}).get("windows", "<absent>"), 0)
+ check("empty/the caller is told", [w.category.__name__ for w in caught], ["RuntimeWarning"])
+
+ # 8d. the fits-in-one-window path answers to a hook too, so 0 keeps meaning "no window scored this"
+ a = make_real_agent()
+ short_hooked, exc = _attempt(
+ lambda: a.predict_long({"body": "x" * 50}, Q, on_predict_start=lambda ctx: ctx.skip([DOC])))
+ check("short skip/no forward pass", a._forward_calls, [])
+ check("short skip/a hook answer is not counted as a window read",
+ ((short_hooked or {}).get("usage") or {}).get("windows", "<absent>"), 0)
+ a = make_real_agent()
+ short_plain, exc = _attempt(lambda: a.predict_long({"body": "x" * 50}, Q))
+ check("short skip/the same state without a hook is one window",
+ ((short_plain or {}).get("usage") or {}).get("windows", "<absent>"), 1)
+
+ # 9. a batch call that returns the wrong count for its own states is an error, not a guess
+ a = make_agent(lambda s, q: [{"answers": {}, "usage": {}}] * 2)
+ check_raises("count/rejects a result count matching nothing", ValueError,
+ lambda: a.predict_long(LONG, Q))
+
+ # 10. the count is total, so one question works on every result predict_long can return
+ short_all = make_agent(lambda s, q: []).predict_long({"body": "x" * 50}, Q)
+ scan_agent = make_agent(canned)
+ scanned_all = scan_agent.predict_long(LONG, Q)
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ hook_all, hook_exc = _attempt(
+ lambda: make_real_agent().predict_long(LONG, Q, on_predict_start=lambda ctx: ctx.skip([DOC])))
+ check("total/a hook answer returns rather than being refused", _kind(hook_exc), None)
+ check("total/no path leaves the key out",
+ [(r or {}).get("usage", {}).get("windows", "<absent>")
+ for r in (short_all, scanned_all, hook_all)],
+ [1, len(scan_agent._calls["batch_states"]), 0])
+ check("total/0 separates a hook answer from a single-window one",
+ ((hook_all or {}).get("usage", {}).get("windows", "<absent>"),
+ short_all["usage"].get("windows", "<absent>")),
+ (0, 1))
+
+
+ # 11. the page that teaches this key teaches the value the code actually writes
+ README = open(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+ "README.md"), encoding="utf-8").read()
+ bullet = README[README.index("A state that already fits one window"):][:600]
+ check("docs/README pins the single-window value", 'usage["windows"] = 1' in bullet, True)
+ check("docs/README states the key is total", "The key is total" in bullet, True)
+ check("docs/README documents all three counts", sorted(set(re.findall(r"`([0-9N])`", bullet))),
+ ["0", "1", "N"])
+ # The two rules this branch added, pinned where they are taught rather than in the code comment
+ hooks_bullet = README[README.index("Hooks wrap the inference that answers the state"):][:900]
+ check("docs/README says attribution survives only an unchanged scan",
+ "reported when the scan reached inference unchanged" in hooks_bullet, True)
+ check("docs/README names ctx.skip as the way to answer", "ctx.skip(...)" in hooks_bullet, True)
+ API = open(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+ "docs", "hooks", "api.md"), encoding="utf-8").read()
+ api_para = API[API.index("On `predict_long` the hooks wrap"):][:1600]
+ check("docs/api.md tabulates the rewritten scan as its own case",
+ "| replaced the scan, in any way |" in api_para, True)
+ check("docs/api.md tabulates a hook answer as zero windows",
+ "| answered with `ctx.skip([result])` | `0` |" in api_para, True)
+ check("docs/predict_long's docstring says the key is total",
+ "always present" in (Agent.predict_long.__doc__ or ""), True)
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL " + f)