Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(hooks): guard run_coroutine_sync, validate hooks_timeout, carry context

Emiliano Gandini Outeda141ab0eea944
diff --git a/docs/hooks/api.md b/docs/hooks/api.md
index 51fbc7be..8817e666 100644
--- a/docs/hooks/api.md
+++ b/docs/hooks/api.md
@@ -188,7 +188,9 @@ Where the coroutine runs:
- If the calling thread has no running loop, it is run with `asyncio.run`.
- If it already has one (a caller inside an async function), it runs on a dedicated background
loop, so the calling thread can block without deadlocking. Pass `AsyncHook(hook, loop=...)` to
- funnel onto a specific loop; it must not be running in the calling thread.
+ funnel onto a specific loop; it must be running, and must not be the calling thread's own loop.
+ Both are checked: a stopped loop and the caller's own loop each raise `ValueError` instead of
+ blocking forever.
A hook with no `async` methods is unaffected.
diff --git a/docs/hooks/errors.md b/docs/hooks/errors.md
index 7b7825e3..8dcde7be 100644
--- a/docs/hooks/errors.md
+++ b/docs/hooks/errors.md
@@ -135,12 +135,19 @@ laya.load("convaiinnovations/laya", on_predict_end=metrics, hooks_timeout=2.0)
```
It can be set per instance or overridden per call on `predict_batch`, `system_one`,
-`Router.route`, `Router.predict` and `ONNXAgent.system_one`.
+`Router.route`, `Router.predict` and `ONNXAgent.system_one`. The value must be positive; `0` or a
+negative number raises `ValueError` at the point it is set, rather than racing on a zero-length
+`join`.
+
+A timed hook runs on a worker thread in a copy of the caller's `contextvars` context, so a
+request id or tracing span set by the caller is visible to the hook.
One honest caveat: Python cannot interrupt a thread, so a timed-out hook keeps running in the
background. The timeout bounds how long the request waits, not how long the hook lives. Use it to
keep a served request responsive, not to reclaim the work. For a hook that can hang, also give the
-underlying call its own timeout (a socket or HTTP timeout).
+underlying call its own timeout (a socket or HTTP timeout). Because the thread cannot be
+reclaimed, a hook that hangs on every call grows threads one per call; give a hook that can hang
+its own bound rather than relying on `hooks_timeout` to stop it.
For an async hook, the coroutine runs on the event loop; a timeout on the calling side still
returns after the limit, and the coroutine keeps running on the loop.
diff --git a/laya/agent.py b/laya/agent.py
index 31b9376a..7e779f03 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -27,7 +27,10 @@ from .common import (
serialize_state,
temp_bucket,
)
-from .hooks import HookRegistry, PredictContext, aggregate_usage, compose_hooks, dispatch, normalise_hooks
+from .hooks import (
+ HookRegistry, PredictContext, aggregate_usage, compose_hooks, dispatch, normalise_hooks,
+ validate_timeout,
+)
def _fix_tokenizer_config(path: str):
@@ -242,7 +245,7 @@ class Agent(HookRegistry):
self.hooks = normalise_hooks(hooks, on_predict_start, on_predict_end)
self.hooks_raise = bool(hooks_raise)
self.hooks_concurrent = bool(hooks_concurrent)
- self.hooks_timeout = None if hooks_timeout is None else float(hooks_timeout)
+ self.hooks_timeout = None if hooks_timeout is None else validate_timeout(hooks_timeout)
self._hooks_lock = threading.RLock() if not hooks_concurrent else None
self._hooks_mutex = threading.Lock()
self.model_id = model_id_or_path
@@ -760,7 +763,7 @@ class Agent(HookRegistry):
"""
active = compose_hooks(self.hooks, hooks, on_predict_start, on_predict_end)
raise_errors = self.hooks_raise if hooks_raise is None else bool(hooks_raise)
- timeout = self.hooks_timeout if hooks_timeout is None else float(hooks_timeout)
+ timeout = self.hooks_timeout if hooks_timeout is None else validate_timeout(hooks_timeout)
ctx = PredictContext(states=states, questions=questions, model=self.model_id, agent=self,
max_len=max_len, head_max_len=head_max_len)
try:
diff --git a/laya/hooks.py b/laya/hooks.py
index bae6d80b..29a7b1a9 100644
--- a/laya/hooks.py
+++ b/laya/hooks.py
@@ -7,6 +7,7 @@ Everything here is pure Python: importing `laya` must not start pulling torch.
from __future__ import annotations
import asyncio
+import contextvars
import inspect
import threading
import time
@@ -217,6 +218,21 @@ def compose_hooks(installed, hooks=None, on_predict_start=None, on_predict_end=N
return default_hooks() + list(installed) + normalise_hooks(hooks, on_predict_start, on_predict_end)
+def validate_timeout(value: Optional[float]) -> Optional[float]:
+ """Return *value* as a positive float, or ``None`` for no limit.
+
+ A non-positive timeout is rejected here rather than left to
+ ``thread.join``: ``join(0)`` and ``join(-1)`` return before the hook has
+ started, so the outcome of a fast hook with such a value is a race.
+ """
+ if value is None:
+ return None
+ timeout = float(value)
+ if timeout <= 0:
+ raise ValueError("hooks_timeout must be a positive number or None; got %r" % (value,))
+ return timeout
+
+
_BACKGROUND_LOOP: Optional[asyncio.AbstractEventLoop] = None
_BACKGROUND_LOOP_LOCK = threading.Lock()
@@ -239,10 +255,24 @@ def run_coroutine_sync(coro: Awaitable[Any], loop: Optional[asyncio.AbstractEven
Uses `asyncio.run` when the calling thread has no running loop. When it does (a caller
inside an async function, or a framework that already runs a loop), the coroutine is run on
a dedicated background loop so the calling thread can block on it without deadlocking. Pass
- `loop` to use a specific loop instead of the background one; it must not be running in the
- calling thread.
+ `loop` to use a specific loop instead of the background one.
+
+ A supplied `loop` must be running somewhere, and must not be the calling thread's own
+ loop. Both are checked: the first would otherwise block forever with no coroutine ever
+ scheduled, and the second would block the only thread that could run the coroutine.
"""
if loop is not None:
+ if not loop.is_running():
+ raise ValueError("run_coroutine_sync: the loop passed is not running")
+ try:
+ running = asyncio.get_running_loop()
+ except RuntimeError:
+ running = None
+ if loop is running:
+ raise ValueError(
+ "run_coroutine_sync: the loop passed is running in the calling thread; "
+ "blocking on it would deadlock"
+ )
return asyncio.run_coroutine_threadsafe(coro, loop).result()
try:
asyncio.get_running_loop()
@@ -275,6 +305,11 @@ class AsyncHook:
def __init__(self, hook: Any, loop: Optional[asyncio.AbstractEventLoop] = None):
if isinstance(hook, type):
raise TypeError("AsyncHook wraps an instance, not a class; got %s" % hook.__name__)
+ if not any(hasattr(hook, event) for event in HOOK_EVENTS):
+ raise TypeError(
+ "AsyncHook wraps an object implementing at least one of %s; got %s"
+ % (", ".join(HOOK_EVENTS), type(hook).__name__)
+ )
self.hook = hook
self.loop = loop
@@ -378,6 +413,8 @@ def _hook_name(method: Any) -> str:
def _call_hook(method: Any, ctx: PredictContext, timeout: Optional[float]) -> None:
"""Call one hook method, running its result if it is awaitable, under an optional timeout."""
+ timeout = validate_timeout(timeout)
+
def invoke():
result = method(ctx)
if inspect.isawaitable(result):
@@ -389,9 +426,14 @@ def _call_hook(method: Any, ctx: PredictContext, timeout: Optional[float]) -> No
box: List[BaseException] = []
+ # Run the hook in a copy of the caller's context, so a `contextvars` value (a request id,
+ # a tracing span) set by the caller is visible to the hook even though it runs on another
+ # thread. The no-timeout path runs inline and inherits the context already.
+ context = contextvars.copy_context()
+
def runner():
try:
- invoke()
+ context.run(invoke)
except BaseException as exc: # noqa: BLE001 -- re-raised on the caller thread
box.append(exc)
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py
index 5622f19a..46df9b1d 100644
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@ -7,7 +7,10 @@ from typing import Any, Dict, Optional, Union
import numpy as np
-from laya.hooks import HookRegistry, PredictContext, aggregate_usage, compose_hooks, dispatch, normalise_hooks
+from laya.hooks import (
+ HookRegistry, PredictContext, aggregate_usage, compose_hooks, dispatch, normalise_hooks,
+ validate_timeout,
+)
from laya.common import (
QTYPES,
answer_confidence,
@@ -63,7 +66,7 @@ class ONNXAgent(HookRegistry):
self.hooks = normalise_hooks(hooks, on_predict_start, on_predict_end)
self.hooks_raise = bool(hooks_raise)
self.hooks_concurrent = bool(hooks_concurrent)
- self.hooks_timeout = None if hooks_timeout is None else float(hooks_timeout)
+ self.hooks_timeout = None if hooks_timeout is None else validate_timeout(hooks_timeout)
self._hooks_lock = threading.RLock() if not hooks_concurrent else None
self._hooks_mutex = threading.Lock()
self.model_id = model_id_or_path
@@ -192,7 +195,7 @@ class ONNXAgent(HookRegistry):
"""Evaluate typed questions, running any opt-in hooks around the inference."""
active = compose_hooks(self.hooks, hooks, on_predict_start, on_predict_end)
raise_errors = self.hooks_raise if hooks_raise is None else bool(hooks_raise)
- timeout = self.hooks_timeout if hooks_timeout is None else float(hooks_timeout)
+ timeout = self.hooks_timeout if hooks_timeout is None else validate_timeout(hooks_timeout)
ctx = PredictContext(states=[state], questions=questions, model=self.model_id, agent=self,
max_len=max_len, head_max_len=head_max_len)
try:
diff --git a/laya/router.py b/laya/router.py
index c6fa2571..3adb3787 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -35,7 +35,10 @@ import time
from collections.abc import Sequence as SequenceABC
from typing import Any, Dict, List, Optional, Sequence, Union
-from .hooks import HookRegistry, PredictContext, aggregate_usage, compose_hooks, dispatch, normalise_hooks
+from .hooks import (
+ HookRegistry, PredictContext, aggregate_usage, compose_hooks, dispatch, normalise_hooks,
+ validate_timeout,
+)
from .lang import analyse
# The hub repo bundles all three checkpoints; only the requested subfolder is downloaded.
@@ -231,7 +234,7 @@ class Router(HookRegistry):
self.hooks = normalise_hooks(hooks, on_predict_start, on_predict_end)
self.hooks_raise = bool(hooks_raise)
self.hooks_concurrent = bool(hooks_concurrent)
- self.hooks_timeout = None if hooks_timeout is None else float(hooks_timeout)
+ self.hooks_timeout = None if hooks_timeout is None else validate_timeout(hooks_timeout)
self._hooks_lock = threading.RLock() if not hooks_concurrent else None
self._hooks_mutex = threading.Lock()
self.models = dict(STANDALONE_MODELS if standalone_repos else DEFAULT_MODELS)
@@ -419,7 +422,7 @@ class Router(HookRegistry):
decision = self._route(state, questions, model=model, task=task, lang=lang, lang_guess=lang_guess)
raise_errors = self.hooks_raise if hooks_raise is None else bool(hooks_raise)
active = compose_hooks(self.hooks, hooks)
- timeout = self.hooks_timeout if hooks_timeout is None else float(hooks_timeout)
+ timeout = self.hooks_timeout if hooks_timeout is None else validate_timeout(hooks_timeout)
ctx = PredictContext(states=[state], questions=questions or {}, decision=decision, router=self)
dispatch(active, "on_route", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
return ctx.decision
@@ -543,7 +546,7 @@ class Router(HookRegistry):
"""
active = compose_hooks(self.hooks, hooks, on_predict_start, on_predict_end)
raise_errors = self.hooks_raise if hooks_raise is None else bool(hooks_raise)
- timeout = self.hooks_timeout if hooks_timeout is None else float(hooks_timeout)
+ timeout = self.hooks_timeout if hooks_timeout is None else validate_timeout(hooks_timeout)
# Per-call hooks apply to the whole call, including on_route inside route().
decision = self.route(state, questions, model=model, task=task, lang=lang,
diff --git a/tests/test_hooks.py b/tests/test_hooks.py
index 3ba09479..f1e290fc 100644
--- a/tests/test_hooks.py
+++ b/tests/test_hooks.py
@@ -1117,6 +1117,59 @@ res = f.predict_batch(["s0"], QUESTIONS, on_predict_end=lambda ctx: None, hooks_
check("timeout/fast hook unaffected", len(res), 1)
+# --------------------------------------------------------------- input validation and context
+import contextvars # noqa: E402
+
+from laya.hooks import validate_timeout # noqa: E402
+
+check_raises("timeout/zero is rejected", ValueError, lambda: validate_timeout(0))
+check_raises("timeout/negative is rejected", ValueError, lambda: validate_timeout(-0.5))
+check("timeout/positive passes through", validate_timeout(1.5), 1.5)
+check("timeout/None means no limit", validate_timeout(None), None)
+
+f = make_fake()
+check_raises("timeout/zero per call is rejected", ValueError,
+ lambda: f.predict_batch(["s0"], QUESTIONS, hooks_timeout=0))
+
+not_running = asyncio.new_event_loop()
+try:
+ check_raises("async/a non-running loop is rejected", ValueError,
+ lambda: run_coroutine_sync(_seven(), loop=not_running))
+finally:
+ not_running.close()
+
+
+async def _own_loop():
+ own = asyncio.get_running_loop()
+ try:
+ run_coroutine_sync(_seven(), loop=own)
+ except ValueError:
+ return "raised"
+ return "no"
+
+
+check("async/the calling thread's own loop is rejected", asyncio.run(_own_loop()), "raised")
+check_raises("async/AsyncHook rejects an object with no events", TypeError,
+ lambda: AsyncHook(object()))
+
+_cv_seen = contextvars.ContextVar("cv_seen", default=None)
+_cv_calls = []
+
+
+class CvHook:
+ def on_predict_start(self, ctx):
+ _cv_calls.append(_cv_seen.get())
+
+
+f = make_fake()
+_token = _cv_seen.set("request-1")
+try:
+ f.predict_batch(["s0"], QUESTIONS, hooks=[CvHook()], hooks_timeout=1.0)
+finally:
+ _cv_seen.reset(_token)
+check("timeout/a timed hook sees the caller's contextvars", _cv_calls, ["request-1"])
+
+
# --------------------------------------------------------------- report
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f_ in FAIL: