Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(hooks): async hooks and per-hook timeout

Emiliano Gandini Outeda630b908858df
diff --git a/docs/hooks/api.md b/docs/hooks/api.md
index 519e5559..51fbc7be 100644
--- a/docs/hooks/api.md
+++ b/docs/hooks/api.md
@@ -165,9 +165,36 @@ Defaults apply to every event, including the Router lifecycle events `on_load` a
The registry is read at call time, so hooks set after an `Agent` or `Router` is built still apply.
There is no per-instance opt-out; call `clear_default_hooks()` to turn the process-wide set off.
+## Async hooks
+
+An event may be a coroutine. Wrap the hook in `AsyncHook` and its `async def` methods run to
+completion in the sync core:
+
+```python
+from laya import AsyncHook
+
+class Remote:
+ async def on_predict_end(self, ctx):
+ await ship(ctx.results)
+
+agent = laya.load("convaiinnovations/laya", hooks=[AsyncHook(Remote())])
+```
+
+A plain async callable passed to `on_predict_start=` / `on_predict_end=` also works, because
+`dispatch` runs any awaitable a hook returns.
+
+Where the coroutine runs:
+
+- If the calling thread has no running loop, it is run with `asyncio.run`.
+- If it already has one (a caller inside an async function), it runs on a dedicated background
+ loop, so the calling thread can block without deadlocking. Pass `AsyncHook(hook, loop=...)` to
+ funnel onto a specific loop; it must not be running in the calling thread.
+
+A hook with no `async` methods is unaffected.
+
## Configuration surface
-Every entry point accepts the same five hook parameters. `hooks` takes an object or a sequence
+Every entry point accepts the same hook parameters. `hooks` takes an object or a sequence
of objects; `on_predict_start` / `on_predict_end` take a callable or a sequence.
| parameter | type | default | meaning |
@@ -177,6 +204,7 @@ of objects; `on_predict_start` / `on_predict_end` take a callable or a sequence.
| `on_predict_end` | `PredictHook \| Sequence[PredictHook] \| None` | `None` | convenience callables for one event. |
| `hooks_raise` | `bool` | `True` | `True`: a hook exception propagates. `False`: warn and continue. |
| `hooks_concurrent` | `bool` | `True` | `False`: dispatch hooks under a lock, one at a time. |
+| `hooks_timeout` | `float \| None` | `None` | per-hook time limit in seconds; `None` means no limit. |
### Agent
@@ -185,24 +213,24 @@ Agent(
model_id_or_path="convaiinnovations/laya",
device=None, token=None, subfolder=None, fast=False, compile=False,
hooks=None, on_predict_start=None, on_predict_end=None,
- hooks_raise=True, hooks_concurrent=True,
+ hooks_raise=True, hooks_concurrent=True, hooks_timeout=None,
)
load(..., hooks=None, on_predict_start=None, on_predict_end=None,
- hooks_raise=True, hooks_concurrent=True)
+ hooks_raise=True, hooks_concurrent=True, hooks_timeout=None)
agent.predict_batch(states, questions, batch_size=None,
hooks=None, on_predict_start=None, on_predict_end=None, hooks_raise=None,
- max_len=None, head_max_len=None, sort_by_length=False)
+ hooks_timeout=None, max_len=None, head_max_len=None, sort_by_length=False)
agent.system_one(state, questions,
hooks=None, on_predict_start=None, on_predict_end=None, hooks_raise=None,
- max_len=None, head_max_len=None)
+ hooks_timeout=None, max_len=None, head_max_len=None)
agent.predict(...) # alias of system_one
```
-- `hooks_raise` on a per-call method defaults to `None`, meaning "use the instance value".
+- `hooks_raise` and `hooks_timeout` on a per-call method default to `None`, meaning "use the instance value".
- `hooks_concurrent` is instance-level only.
### Router
@@ -220,7 +248,7 @@ router.route(state, questions=None, model=None, task=None, lang=None, lang_guess
router.predict(state, questions, model=None, task=None, lang=None, lang_guess=None,
hooks=None, on_predict_start=None, on_predict_end=None, hooks_raise=None,
- max_len=None, head_max_len=None)
+ hooks_timeout=None, max_len=None, head_max_len=None)
router.system_one(...) # alias of predict
router.load(name) # builds on first use; fires on_load
@@ -243,7 +271,7 @@ ONNXAgent(model_id_or_path, onnx_path="laya.onnx", subfolder=None,
onnx_agent.system_one(state, questions,
hooks=None, on_predict_start=None, on_predict_end=None, hooks_raise=None,
- max_len=None, head_max_len=None)
+ hooks_timeout=None, max_len=None, head_max_len=None)
onnx_agent.predict(...) # alias of system_one
```
@@ -300,9 +328,16 @@ dispatch(hooks, event, ctx, *, raise_errors=True, lock=None) -> None
aggregate_usage(results) -> {"input_tokens": int, "output_tokens": int}
```
+```python
+dispatch(hooks, event, ctx, *, raise_errors=True, lock=None, timeout=None)
+run_coroutine_sync(coro, loop=None)
+```
+
`normalise_hooks` flattens a `hooks` object/sequence and the two callables into one ordered list.
-`dispatch` calls `event` on every hook that implements it, applying the raise policy and lock.
-`aggregate_usage` sums per-state usage blocks.
+`dispatch` calls `event` on every hook that implements it, applying the raise policy, lock and
+timeout, and runs a hook's result if it is awaitable. `run_coroutine_sync` runs an awaitable to
+completion from sync code, on the caller's loop if it is free, or on a background loop if the
+caller already has one. `aggregate_usage` sums per-state usage blocks.
```python
from laya.hooks import normalise_hooks, dispatch, PredictContext
diff --git a/docs/hooks/errors.md b/docs/hooks/errors.md
index 90e40005..7b7825e3 100644
--- a/docs/hooks/errors.md
+++ b/docs/hooks/errors.md
@@ -124,6 +124,27 @@ laya: hook Metrics.on_predict_end failed: connection reset
The warning is emitted once per failure, not once per hook definition, so a flaky hook under
load can be noisy. Aggregate or rate-limit inside the hook if that matters.
+## Timeouts
+
+`hooks_timeout` bounds each hook call in seconds. A hook still running after the limit is treated
+as a hook failure: `TimeoutError` when `hooks_raise=True`, a `RuntimeWarning` when `False`. `None`
+(the default) means no limit.
+
+```python
+laya.load("convaiinnovations/laya", on_predict_end=metrics, hooks_timeout=2.0)
+```
+
+It can be set per instance or overridden per call on `predict_batch`, `system_one`,
+`Router.route`, `Router.predict` and `ONNXAgent.system_one`.
+
+One honest caveat: Python cannot interrupt a thread, so a timed-out hook keeps running in the
+background. The timeout bounds how long the request waits, not how long the hook lives. Use it to
+keep a served request responsive, not to reclaim the work. For a hook that can hang, also give the
+underlying call its own timeout (a socket or HTTP timeout).
+
+For an async hook, the coroutine runs on the event loop; a timeout on the calling side still
+returns after the limit, and the coroutine keeps running on the loop.
+
## Choosing a policy
| hook kind | recommended | why |
diff --git a/docs/hooks/examples.md b/docs/hooks/examples.md
index ddfd73b6..b4fef349 100644
--- a/docs/hooks/examples.md
+++ b/docs/hooks/examples.md
@@ -19,6 +19,8 @@ Copy-paste recipes. Every snippet is self-contained apart from the helpers it na
- [ONNXAgent](#onnxagent)
- [Runtime registration](#runtime-registration)
- [Base class and process-wide defaults](#base-class-and-process-wide-defaults)
+- [Async hooks](#async-hooks)
+- [Hook timeout](#hook-timeout)
- [Token budget](#token-budget)
- [Testing hooks](#testing-hooks)
@@ -340,6 +342,46 @@ agent = laya.load("convaiinnovations/laya", on_predict_start=widen)
agent.system_one(state, questions, head_max_len=324, max_len=1024)
```
+## Async hooks
+
+Wrap an async hook in `AsyncHook`; each coroutine runs to completion in the sync core, whether the
+caller is synchronous or already inside an event loop.
+
+```python
+import laya
+from laya import AsyncHook
+
+class RemoteAudit:
+ async def on_predict_end(self, ctx):
+ await ship(ctx.run_id, ctx.results)
+
+agent = laya.load("convaiinnovations/laya", hooks=[AsyncHook(RemoteAudit())])
+```
+
+A plain async callable works too:
+
+```python
+async def async_end(ctx):
+ await ship(ctx.results)
+
+agent.system_one(state, questions, on_predict_end=async_end)
+```
+
+## Hook timeout
+
+Bound each hook call, so a stuck hook cannot hang a served request:
+
+```python
+agent = laya.load("convaiinnovations/laya", on_predict_end=metrics, hooks_timeout=2.0)
+
+# or per call
+agent.system_one(state, questions, on_predict_end=metrics, hooks_timeout=0.5)
+```
+
+A timed-out hook raises `TimeoutError` (or warns when `hooks_raise=False`). The hook keeps running
+in the background, so also give network calls their own timeout. See
+[errors](errors.md#timeouts).
+
## Testing hooks
Assert what a hook saw without a model: drive `predict_batch` with the encode/forward/decode
diff --git a/docs/hooks/index.md b/docs/hooks/index.md
index 9a96e3cc..5103914c 100644
--- a/docs/hooks/index.md
+++ b/docs/hooks/index.md
@@ -124,8 +124,12 @@ automatically. `Agent` hooks fire whenever the Router runs an attached or built
- No hooks configured means no behavioural change. The unset path is regression-tested.
- All hook parameters are keyword arguments with defaults, so existing calls keep working.
- `laya/hooks.py` is pure Python: `import laya` does not pull in torch because of it.
-- Hooks are synchronous. Keep them fast and non-blocking; see
- [errors](errors.md) and [patterns](patterns.md) for the consequences on `laya.serve`.
+- Hooks are synchronous by default. An `async def` event can be wrapped in
+ [`AsyncHook`](api.md#async-hooks), or passed as a plain async callable, and it runs to
+ completion for you.
+- [`hooks_timeout`](errors.md#timeouts) bounds a slow hook so it cannot hang a served request.
+- Keep hooks fast and non-blocking; see [errors](errors.md) and [patterns](patterns.md) for the
+ consequences on `laya.serve`.
## See also
diff --git a/laya/__init__.py b/laya/__init__.py
index 792d9089..cfbbe14a 100644
--- a/laya/__init__.py
+++ b/laya/__init__.py
@@ -1,7 +1,7 @@
"""Laya: Fast, non-autoregressive System 1 decision engine with calibrated probabilities."""
from .email import clean_email_body, email_state
-from .hooks import BaseHook, Hook, PredictContext, PredictHook
+from .hooks import AsyncHook, BaseHook, Hook, PredictContext, PredictHook
from .lang import analyse as detect_language
from .lang import detect_script, is_english
from .presets import (
@@ -95,6 +95,7 @@ __all__ = [
"PredictHook",
"Hook",
"BaseHook",
+ "AsyncHook",
"decide",
"DecisionResult",
"__version__",
diff --git a/laya/agent.py b/laya/agent.py
index a76041f7..31b9376a 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -199,6 +199,7 @@ class Agent(HookRegistry):
# hand-built instance (`Agent.__new__` in tests) working and make an unset hook a no-op.
hooks_raise = True
hooks_concurrent = True
+ hooks_timeout = None
_hooks_lock = None
model_id = None
# Autocast is chosen per device in __init__; this default covers instances built
@@ -222,6 +223,7 @@ class Agent(HookRegistry):
on_predict_end=None,
hooks_raise: bool = True,
hooks_concurrent: bool = True,
+ hooks_timeout: Optional[float] = None,
):
"""Load a Laya checkpoint.
@@ -233,12 +235,14 @@ class Agent(HookRegistry):
downloaded, so bundling does not cost every user the whole family.
`hooks` / `on_predict_start` / `on_predict_end` observe or shape every prediction; see
- `laya.hooks`. `hooks_raise=False` warns and continues when a hook fails, and
- `hooks_concurrent=False` serialises hooks that are not safe to run in parallel.
+ `laya.hooks`. `hooks_raise=False` warns and continues when a hook fails,
+ `hooks_concurrent=False` serialises hooks that are not safe to run in parallel, and
+ `hooks_timeout` bounds each hook call in seconds (None means no limit).
"""
self.hooks = normalise_hooks(hooks, on_predict_start, on_predict_end)
self.hooks_raise = bool(hooks_raise)
self.hooks_concurrent = bool(hooks_concurrent)
+ self.hooks_timeout = None if hooks_timeout is None else float(hooks_timeout)
self._hooks_lock = threading.RLock() if not hooks_concurrent else None
self._hooks_mutex = threading.Lock()
self.model_id = model_id_or_path
@@ -716,6 +720,7 @@ class Agent(HookRegistry):
hooks=None,
on_predict_start=None, on_predict_end=None,
hooks_raise: Optional[bool] = None,
+ hooks_timeout: Optional[float] = None,
max_len: Optional[int] = None,
head_max_len: Optional[int] = None,
sort_by_length: bool = False) -> List[Dict[str, Any]]:
@@ -738,6 +743,7 @@ class Agent(HookRegistry):
state/questions or call `ctx.skip(...)` to short-circuit inference.
on_predict_end (PredictHookArg): A per-call end hook. It may rewrite the results.
hooks_raise: Override the Agent's `hooks_raise` for this call.
+ hooks_timeout: Override the Agent's `hooks_timeout` for this call.
max_len: Override the agent config's `max_len` for this call. A start hook may also
set `ctx.max_len` to shape the token budget.
head_max_len: Override the agent config's `head_max_len` for this call. A start hook
@@ -754,10 +760,11 @@ class Agent(HookRegistry):
"""
active = compose_hooks(self.hooks, hooks, on_predict_start, on_predict_end)
raise_errors = self.hooks_raise if hooks_raise is None else bool(hooks_raise)
+ timeout = self.hooks_timeout if hooks_timeout is None else float(hooks_timeout)
ctx = PredictContext(states=states, questions=questions, model=self.model_id, agent=self,
max_len=max_len, head_max_len=head_max_len)
try:
- dispatch(active, "on_predict_start", ctx, raise_errors=raise_errors, lock=self._hooks_lock)
+ dispatch(active, "on_predict_start", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
states, questions = ctx.states, ctx.questions
if ctx.results is None:
# A start hook may have normalised a bare string/dict into a list; only the value
@@ -828,7 +835,7 @@ class Agent(HookRegistry):
except BaseException as exc:
ctx.error = exc
try:
- dispatch(active, "on_error", ctx, raise_errors=raise_errors, lock=self._hooks_lock)
+ dispatch(active, "on_error", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
except BaseException as hook_exc:
# A failing on_error hook must not hide the failure that triggered it.
exc.__context__ = hook_exc
@@ -838,7 +845,7 @@ class Agent(HookRegistry):
if ctx.results is not None:
ctx.usage = aggregate_usage(ctx.results)
try:
- dispatch(active, "on_predict_end", ctx, raise_errors=raise_errors, lock=self._hooks_lock)
+ dispatch(active, "on_predict_end", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
except BaseException as hook_exc:
# End hooks run on the failure path too; do not let one mask the real error.
if ctx.error is not None:
@@ -851,6 +858,7 @@ class Agent(HookRegistry):
def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]], lang: Optional[str] = None,
hooks=None, on_predict_start=None, on_predict_end=None,
hooks_raise: Optional[bool] = None,
+ hooks_timeout: Optional[float] = None,
max_len: Optional[int] = None,
head_max_len: Optional[int] = None) -> Dict[str, Any]:
"""Evaluate typed questions across state in a single, parallel forward pass.
@@ -878,6 +886,7 @@ class Agent(HookRegistry):
return self.predict_batch([state], questions, lang=lang, hooks=hooks,
on_predict_start=on_predict_start,
on_predict_end=on_predict_end, hooks_raise=hooks_raise,
+ hooks_timeout=hooks_timeout,
max_len=max_len, head_max_len=head_max_len)[0]
def __enter__(self):
@@ -922,7 +931,8 @@ def load(model_id_or_path: str = "convaiinnovations/laya", device: Optional[str]
token: Optional[str] = None, subfolder: Optional[str] = None, fast: bool = False,
lang_temperatures: Optional[Dict[str, Dict[str, Any]]] = None,
hooks=None, on_predict_start=None, on_predict_end=None,
- hooks_raise: bool = True, hooks_concurrent: bool = True) -> Agent:
+ hooks_raise: bool = True, hooks_concurrent: bool = True,
+ hooks_timeout: Optional[float] = None) -> Agent:
"""Load a Laya agent.
`subfolder` picks one checkpoint out of a repo that bundles several:
@@ -937,4 +947,5 @@ def load(model_id_or_path: str = "convaiinnovations/laya", device: Optional[str]
return Agent(model_id_or_path, device=device, token=token, subfolder=subfolder, fast=fast,
lang_temperatures=lang_temperatures,
hooks=hooks, on_predict_start=on_predict_start, on_predict_end=on_predict_end,
- hooks_raise=hooks_raise, hooks_concurrent=hooks_concurrent)
+ hooks_raise=hooks_raise, hooks_concurrent=hooks_concurrent,
+ hooks_timeout=hooks_timeout)
diff --git a/laya/hooks.py b/laya/hooks.py
index 27f419fc..bae6d80b 100644
--- a/laya/hooks.py
+++ b/laya/hooks.py
@@ -6,13 +6,15 @@ Everything here is pure Python: importing `laya` must not start pulling torch.
"""
from __future__ import annotations
+import asyncio
+import inspect
import threading
import time
import uuid
import warnings
from contextlib import contextmanager
from dataclasses import dataclass, field
-from typing import Any, Callable, Dict, List, Optional, Protocol, Sequence, Union
+from typing import Any, Awaitable, Callable, Dict, List, Optional, Protocol, Sequence, Union
@dataclass(eq=False)
@@ -109,7 +111,7 @@ class _StartAdapter:
self.fn = fn
def on_predict_start(self, ctx: PredictContext) -> None:
- self.fn(ctx)
+ return self.fn(ctx)
class _EndAdapter:
@@ -123,7 +125,7 @@ class _EndAdapter:
self.fn = fn
def on_predict_end(self, ctx: PredictContext) -> None:
- self.fn(ctx)
+ return self.fn(ctx)
def _as_sequence(value: Any) -> tuple:
@@ -215,6 +217,97 @@ def compose_hooks(installed, hooks=None, on_predict_start=None, on_predict_end=N
return default_hooks() + list(installed) + normalise_hooks(hooks, on_predict_start, on_predict_end)
+_BACKGROUND_LOOP: Optional[asyncio.AbstractEventLoop] = None
+_BACKGROUND_LOOP_LOCK = threading.Lock()
+
+
+def _background_loop() -> asyncio.AbstractEventLoop:
+ """A daemon event loop thread, for running coroutines when the caller already has a loop."""
+ global _BACKGROUND_LOOP
+ with _BACKGROUND_LOOP_LOCK:
+ if _BACKGROUND_LOOP is None or not _BACKGROUND_LOOP.is_running():
+ _BACKGROUND_LOOP = asyncio.new_event_loop()
+ thread = threading.Thread(target=_BACKGROUND_LOOP.run_forever, daemon=True,
+ name="laya-async-hooks")
+ thread.start()
+ return _BACKGROUND_LOOP
+
+
+def run_coroutine_sync(coro: Awaitable[Any], loop: Optional[asyncio.AbstractEventLoop] = None) -> Any:
+ """Run an awaitable to completion from synchronous code.
+
+ Uses `asyncio.run` when the calling thread has no running loop. When it does (a caller
+ inside an async function, or a framework that already runs a loop), the coroutine is run on
+ a dedicated background loop so the calling thread can block on it without deadlocking. Pass
+ `loop` to use a specific loop instead of the background one; it must not be running in the
+ calling thread.
+ """
+ if loop is not None:
+ return asyncio.run_coroutine_threadsafe(coro, loop).result()
+ try:
+ asyncio.get_running_loop()
+ except RuntimeError:
+ return asyncio.run(coro)
+ return asyncio.run_coroutine_threadsafe(coro, _background_loop()).result()
+
+
+class AsyncHook:
+ """Wrap an async hook so its coroutine event methods run to completion in the sync core.
+
+ The wrapped object may implement any subset of the events as `async def` methods, or as
+ plain callables that return awaitables. Each event is run with `run_coroutine_sync`, so it
+ works whether the caller is synchronous or already inside an event loop.
+
+ from laya import AsyncHook
+
+ class Remote(BaseHook):
+ async def on_predict_end(self, ctx):
+ await ship(ctx.results)
+
+ agent = laya.load("convaiinnovations/laya", hooks=[AsyncHook(Remote())])
+
+ Pass `loop` to funnel every coroutine onto a specific loop; otherwise a background loop is
+ started on demand when the caller already has one.
+ """
+
+ __slots__ = ("hook", "loop")
+
+ def __init__(self, hook: Any, loop: Optional[asyncio.AbstractEventLoop] = None):
+ if isinstance(hook, type):
+ raise TypeError("AsyncHook wraps an instance, not a class; got %s" % hook.__name__)
+ self.hook = hook
+ self.loop = loop
+
+ def _run(self, event: str, ctx: PredictContext) -> None:
+ method = getattr(self.hook, event, None)
+ if method is None:
+ return
+ result = method(ctx)
+ if inspect.isawaitable(result):
+ run_coroutine_sync(result, loop=self.loop)
+
+ def on_predict_start(self, ctx: PredictContext) -> None:
+ self._run("on_predict_start", ctx)
+
+ def on_predict_end(self, ctx: PredictContext) -> None:
+ self._run("on_predict_end", ctx)
+
+ def on_route(self, ctx: PredictContext) -> None:
+ self._run("on_route", ctx)
+
+ def on_load(self, ctx: PredictContext) -> None:
+ self._run("on_load", ctx)
+
+ def on_evict(self, ctx: PredictContext) -> None:
+ self._run("on_evict", ctx)
+
+ def on_error(self, ctx: PredictContext) -> None:
+ self._run("on_error", ctx)
+
+ def __repr__(self) -> str:
+ return "AsyncHook(%r)" % (self.hook,)
+
+
class HookRegistry:
"""Mixin giving a runtime-mutable hook list.
@@ -278,6 +371,40 @@ def aggregate_usage(results: Sequence[Dict[str, Any]]) -> Dict[str, int]:
return {"input_tokens": total("input_tokens"), "output_tokens": total("output_tokens")}
+def _hook_name(method: Any) -> str:
+ return (getattr(method, "__qualname__", None) or getattr(method, "__name__", None)
+ or repr(method))
+
+
+def _call_hook(method: Any, ctx: PredictContext, timeout: Optional[float]) -> None:
+ """Call one hook method, running its result if it is awaitable, under an optional timeout."""
+ def invoke():
+ result = method(ctx)
+ if inspect.isawaitable(result):
+ run_coroutine_sync(result)
+
+ if timeout is None:
+ invoke()
+ return
+
+ box: List[BaseException] = []
+
+ def runner():
+ try:
+ invoke()
+ except BaseException as exc: # noqa: BLE001 -- re-raised on the caller thread
+ box.append(exc)
+
+ thread = threading.Thread(target=runner, daemon=True, name="laya-hook-timeout")
+ thread.start()
+ thread.join(timeout)
+ if thread.is_alive():
+ # The thread keeps running: Python cannot interrupt it. The timeout bounds the request.
+ raise TimeoutError("laya: hook %s exceeded %gs" % (_hook_name(method), timeout))
+ if box:
+ raise box[0]
+
+
def dispatch(
hooks: Sequence[Any],
event: str,
@@ -285,11 +412,16 @@ def dispatch(
*,
raise_errors: bool = True,
lock: Any = None,
+ timeout: Optional[float] = None,
) -> None:
"""Call `event` on every hook that implements it.
`raise_errors=False` warns and continues, for hooks (telemetry) that must not fail a
request. `lock` serialises dispatch for hooks that are not safe to run concurrently.
+ `timeout` bounds each hook call in seconds; an overrunning hook raises `TimeoutError`, or
+ warns when `raise_errors` is False. Python threads cannot be interrupted, so an overrunning
+ hook keeps running in the background: the timeout protects the request, not the process.
+ A hook that returns an awaitable is run to completion before moving on.
"""
for hook in hooks:
method = getattr(hook, event, None)
@@ -298,9 +430,9 @@ def dispatch(
try:
if lock is not None:
with lock:
- method(ctx)
+ _call_hook(method, ctx, timeout)
else:
- method(ctx)
+ _call_hook(method, ctx, timeout)
except Exception as exc: # noqa: BLE001 -- policy depends on raise_errors
if raise_errors:
raise
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py
index 52326404..5622f19a 100644
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@ -30,6 +30,7 @@ class ONNXAgent(HookRegistry):
# `hooks`/`_hooks_mutex` come from HookRegistry.
hooks_raise = True
hooks_concurrent = True
+ hooks_timeout = None
_hooks_lock = None
model_id = None
@@ -43,6 +44,7 @@ class ONNXAgent(HookRegistry):
on_predict_end=None,
hooks_raise: bool = True,
hooks_concurrent: bool = True,
+ hooks_timeout: Optional[float] = None,
):
"""Load a Laya agent backed by ONNX Runtime.
@@ -56,10 +58,12 @@ class ONNXAgent(HookRegistry):
on_predict_end (PredictHookArg): An opt-in end hook, run after inference.
hooks_raise: When False, a failing hook warns and inference continues.
hooks_concurrent: When False, hooks are serialised with a lock.
+ hooks_timeout: Bounds each hook call in seconds; None means no limit.
"""
self.hooks = normalise_hooks(hooks, on_predict_start, on_predict_end)
self.hooks_raise = bool(hooks_raise)
self.hooks_concurrent = bool(hooks_concurrent)
+ self.hooks_timeout = None if hooks_timeout is None else float(hooks_timeout)
self._hooks_lock = threading.RLock() if not hooks_concurrent else None
self._hooks_mutex = threading.Lock()
self.model_id = model_id_or_path
@@ -182,15 +186,17 @@ class ONNXAgent(HookRegistry):
def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]],
hooks=None, on_predict_start=None, on_predict_end=None,
hooks_raise: Optional[bool] = None,
+ hooks_timeout: Optional[float] = None,
max_len: Optional[int] = None,
head_max_len: Optional[int] = None) -> Dict[str, Any]:
"""Evaluate typed questions, running any opt-in hooks around the inference."""
active = compose_hooks(self.hooks, hooks, on_predict_start, on_predict_end)
raise_errors = self.hooks_raise if hooks_raise is None else bool(hooks_raise)
+ timeout = self.hooks_timeout if hooks_timeout is None else float(hooks_timeout)
ctx = PredictContext(states=[state], questions=questions, model=self.model_id, agent=self,
max_len=max_len, head_max_len=head_max_len)
try:
- dispatch(active, "on_predict_start", ctx, raise_errors=raise_errors, lock=self._hooks_lock)
+ dispatch(active, "on_predict_start", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
if ctx.results is None:
overrides = {}
if ctx.max_len is not None:
@@ -201,7 +207,7 @@ class ONNXAgent(HookRegistry):
except BaseException as exc:
ctx.error = exc
try:
- dispatch(active, "on_error", ctx, raise_errors=raise_errors, lock=self._hooks_lock)
+ dispatch(active, "on_error", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
except BaseException as hook_exc:
exc.__context__ = hook_exc
raise
@@ -210,7 +216,7 @@ class ONNXAgent(HookRegistry):
if ctx.results is not None:
ctx.usage = aggregate_usage(ctx.results)
try:
- dispatch(active, "on_predict_end", ctx, raise_errors=raise_errors, lock=self._hooks_lock)
+ dispatch(active, "on_predict_end", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
except BaseException as hook_exc:
if ctx.error is not None:
ctx.error.__context__ = hook_exc
diff --git a/laya/router.py b/laya/router.py
index 91920c08..c6fa2571 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -207,6 +207,7 @@ class Router(HookRegistry):
# `hooks`/`_hooks_mutex` come from HookRegistry.
hooks_raise = True
hooks_concurrent = True
+ hooks_timeout = None
_hooks_lock = None
def __init__(
@@ -225,10 +226,12 @@ class Router(HookRegistry):
on_predict_end=None,
hooks_raise: bool = True,
hooks_concurrent: bool = True,
+ hooks_timeout: Optional[float] = None,
):
self.hooks = normalise_hooks(hooks, on_predict_start, on_predict_end)
self.hooks_raise = bool(hooks_raise)
self.hooks_concurrent = bool(hooks_concurrent)
+ self.hooks_timeout = None if hooks_timeout is None else float(hooks_timeout)
self._hooks_lock = threading.RLock() if not hooks_concurrent else None
self._hooks_mutex = threading.Lock()
self.models = dict(STANDALONE_MODELS if standalone_repos else DEFAULT_MODELS)
@@ -275,7 +278,7 @@ class Router(HookRegistry):
self._dispatch_lifecycle("on_evict", evicted)
dispatch(compose_hooks(self.hooks), "on_load",
PredictContext(states=[], questions={}, model=key, agent=agent, router=self),
- raise_errors=self.hooks_raise, lock=self._hooks_lock)
+ raise_errors=self.hooks_raise, lock=self._hooks_lock, timeout=self.hooks_timeout)
return agent
def _touch(self, key: str):
@@ -318,7 +321,7 @@ class Router(HookRegistry):
for name in names:
dispatch(compose_hooks(self.hooks), event,
PredictContext(states=[], questions={}, model=name, router=self),
- raise_errors=self.hooks_raise, lock=self._hooks_lock)
+ raise_errors=self.hooks_raise, lock=self._hooks_lock, timeout=self.hooks_timeout)
def attach(self, name: str, agent: Any):
"""Register an already-built Agent under `name` instead of loading a second copy.
@@ -405,6 +408,7 @@ class Router(HookRegistry):
lang_guess: Optional[Any] = None,
hooks=None,
hooks_raise: Optional[bool] = None,
+ hooks_timeout: Optional[float] = None,
) -> RouteDecision:
"""Decide which checkpoint to use, then let `on_route` hooks observe or replace it.
@@ -415,8 +419,9 @@ class Router(HookRegistry):
decision = self._route(state, questions, model=model, task=task, lang=lang, lang_guess=lang_guess)
raise_errors = self.hooks_raise if hooks_raise is None else bool(hooks_raise)
active = compose_hooks(self.hooks, hooks)
+ timeout = self.hooks_timeout if hooks_timeout is None else float(hooks_timeout)
ctx = PredictContext(states=[state], questions=questions or {}, decision=decision, router=self)
- dispatch(active, "on_route", ctx, raise_errors=raise_errors, lock=self._hooks_lock)
+ dispatch(active, "on_route", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
return ctx.decision
def _route(
@@ -525,6 +530,7 @@ class Router(HookRegistry):
on_predict_start=None,
on_predict_end=None,
hooks_raise: Optional[bool] = None,
+ hooks_timeout: Optional[float] = None,
max_len: Optional[int] = None,
head_max_len: Optional[int] = None,
) -> Dict[str, Any]:
@@ -537,10 +543,12 @@ class Router(HookRegistry):
"""
active = compose_hooks(self.hooks, hooks, on_predict_start, on_predict_end)
raise_errors = self.hooks_raise if hooks_raise is None else bool(hooks_raise)
+ timeout = self.hooks_timeout if hooks_timeout is None else float(hooks_timeout)
# Per-call hooks apply to the whole call, including on_route inside route().
decision = self.route(state, questions, model=model, task=task, lang=lang,
- lang_guess=lang_guess, hooks=hooks, hooks_raise=hooks_raise)
+ lang_guess=lang_guess, hooks=hooks, hooks_raise=hooks_raise,
+ hooks_timeout=hooks_timeout)
agent = self.load(decision["model"])
effective_lang = lang
if effective_lang is None and decision.get("detection") and decision["detection"].get("language"):
@@ -550,7 +558,7 @@ class Router(HookRegistry):
model=decision["model"], agent=agent, router=self,
max_len=max_len, head_max_len=head_max_len)
try:
- dispatch(active, "on_predict_start", ctx, raise_errors=raise_errors, lock=self._hooks_lock)
+ dispatch(active, "on_predict_start", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
if ctx.results is None:
# Pass token-budget overrides only when set, so any Agent-like object that does
# not accept them still works on the default path.
@@ -578,7 +586,7 @@ class Router(HookRegistry):
except BaseException as exc:
ctx.error = exc
try:
- dispatch(active, "on_error", ctx, raise_errors=raise_errors, lock=self._hooks_lock)
+ dispatch(active, "on_error", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
except BaseException as hook_exc:
exc.__context__ = hook_exc
raise
@@ -587,7 +595,7 @@ class Router(HookRegistry):
if ctx.results is not None:
ctx.usage = aggregate_usage(ctx.results)
try:
- dispatch(active, "on_predict_end", ctx, raise_errors=raise_errors, lock=self._hooks_lock)
+ dispatch(active, "on_predict_end", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
except BaseException as hook_exc:
if ctx.error is not None:
ctx.error.__context__ = hook_exc
diff --git a/tests/test_hooks.py b/tests/test_hooks.py
index a5a40858..3ba09479 100644
--- a/tests/test_hooks.py
+++ b/tests/test_hooks.py
@@ -1018,6 +1018,105 @@ check("defaults/cover router lifecycle", ld.events,
[("load", "english"), ("evict", "english"), ("load", "multilingual")])
+# --------------------------------------------------------------- async hooks
+import asyncio # noqa: E402
+
+from laya import AsyncHook # noqa: E402
+from laya.hooks import run_coroutine_sync # noqa: E402
+
+calls = []
+
+
+class AsyncAudit:
+ async def on_predict_end(self, ctx):
+ await asyncio.sleep(0)
+ calls.append("wrapped")
+
+
+f = make_fake()
+f.add_hook(AsyncHook(AsyncAudit()))
+f.predict_batch(["s0"], QUESTIONS)
+check("async/wrapped hook awaited", calls, ["wrapped"])
+
+calls.clear()
+
+
+async def async_end(ctx):
+ await asyncio.sleep(0)
+ calls.append("plain")
+
+
+f = make_fake()
+f.predict_batch(["s0"], QUESTIONS, on_predict_end=async_end)
+check("async/plain callable awaited", calls, ["plain"])
+
+calls.clear()
+
+
+async def async_start(ctx):
+ calls.append("loop")
+
+
+f = make_fake()
+
+
+async def _in_loop():
+ f.predict_batch(["s0"], QUESTIONS, on_predict_start=async_start)
+
+
+asyncio.run(_in_loop())
+check("async/works inside a running loop", calls, ["loop"])
+
+
+async def _seven():
+ return 7
+
+
+check("async/run_coroutine_sync returns", run_coroutine_sync(_seven()), 7)
+check_raises("async/AsyncHook rejects a class", TypeError, lambda: AsyncHook(AsyncAudit))
+
+
+# --------------------------------------------------------------- hook timeout
+def slow_hook(ctx):
+ time.sleep(0.3)
+
+
+f = make_fake()
+raised = None
+try:
+ f.predict_batch(["s0"], QUESTIONS, on_predict_start=slow_hook, hooks_timeout=0.05)
+except TimeoutError as exc:
+ raised = str(exc)
+check_true("timeout/raises TimeoutError", isinstance(raised, str) and "exceeded" in raised)
+
+f = make_fake()
+with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ res = f.predict_batch(["s0"], QUESTIONS, on_predict_start=slow_hook,
+ hooks_timeout=0.05, hooks_raise=False)
+check("timeout/hooks_raise=False continues", len(res), 1)
+check_true("timeout/hooks_raise=False warns",
+ any(issubclass(w.category, RuntimeWarning) for w in caught))
+
+f = make_fake()
+f.hooks_timeout = 0.05
+raised = None
+try:
+ f.predict_batch(["s0"], QUESTIONS, on_predict_start=slow_hook)
+except TimeoutError:
+ raised = True
+check("timeout/instance-level applies", raised, True)
+
+f = make_fake()
+f.hooks_timeout = 0.05
+res = f.predict_batch(["s0"], QUESTIONS, on_predict_start=slow_hook, hooks_timeout=5.0)
+check("timeout/per-call override wins", len(res), 1)
+
+f = make_fake()
+res = f.predict_batch(["s0"], QUESTIONS, on_predict_end=lambda ctx: None, hooks_timeout=1.0)
+check("timeout/fast hook unaffected", len(res), 1)
+
+
# --------------------------------------------------------------- report
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f_ in FAIL:
diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py
index 7b66d6ea..5b66c417 100644
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@ -14,7 +14,7 @@ import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import laya # noqa: E402
-from laya import Agent, BaseHook, PredictContext, PredictHook, Router, load # noqa: E402
+from laya import Agent, AsyncHook, BaseHook, PredictContext, PredictHook, Router, load # noqa: E402
from laya.hooks import HOOK_EVENTS, Hook # noqa: E402
from laya.onnx_agent import ONNXAgent # noqa: E402
@@ -56,6 +56,7 @@ HOOK_KEYS = {
"on_predict_end": None,
"hooks_raise": True,
"hooks_concurrent": True,
+ "hooks_timeout": None,
}
# --------------------------------------------------------------- constructors
@@ -76,6 +77,7 @@ for label, fn in (("Agent.predict_batch", Agent.predict_batch),
check_param(label, fn, "on_predict_start", None)
check_param(label, fn, "on_predict_end", None)
check_param(label, fn, "hooks_raise", None)
+ check_param(label, fn, "hooks_timeout", None)
check_param("Agent.predict_batch", Agent.predict_batch, "batch_size", None)
@@ -90,6 +92,7 @@ for label, fn in (("Agent.predict_batch", Agent.predict_batch),
# route() takes per-call hooks so a hook can pin a checkpoint for one call
check_param("Router.route", Router.route, "hooks", None)
check_param("Router.route", Router.route, "hooks_raise", None)
+check_param("Router.route", Router.route, "hooks_timeout", None)
# --------------------------------------------------------------- aliases
check_true("Agent.predict is Agent.system_one", Agent.predict is Agent.system_one)
@@ -119,9 +122,12 @@ for event in HOOK_EVENTS:
check_true("PredictHook is callable-typed", callable(PredictHook))
# --------------------------------------------------------------- exports
-for name in ("PredictContext", "PredictHook", "Hook", "BaseHook"):
+for name in ("PredictContext", "PredictHook", "Hook", "BaseHook", "AsyncHook"):
check_true("__all__/%s" % name, name in laya.__all__)
check_true("laya.%s exists" % name, hasattr(laya, name))
+check_true("laya.hooks/run_coroutine_sync exists",
+ callable(getattr(__import__("laya.hooks", fromlist=["run_coroutine_sync"]),
+ "run_coroutine_sync", None)))
# BaseHook is the concrete no-op base class; all six events exist and are callable.
for event in HOOK_EVENTS:
@@ -138,6 +144,7 @@ for label, cls in (("Agent", Agent), ("Router", Router), ("ONNXAgent", ONNXAgent
check("%s/hooks default" % label, cls.hooks, ())
check("%s/hooks_raise default" % label, cls.hooks_raise, True)
check("%s/hooks_concurrent default" % label, cls.hooks_concurrent, True)
+ check("%s/hooks_timeout default" % label, cls.hooks_timeout, None)
check("%s/_hooks_lock default" % label, cls._hooks_lock, None)
# Agent and ONNXAgent carry the checkpoint id for ctx.model; Router has no single model.