Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(hooks): guard run_coroutine_sync, validate hooks_timeout, carry context
Emiliano Gandini Outeda141ab0eea944
diff --git a/docs/hooks/api.md b/docs/hooks/api.mdindex 51fbc7be..8817e666 100644--- a/docs/hooks/api.md+++ b/docs/hooks/api.md@@ -188,7 +188,9 @@ Where the coroutine runs:- If the calling thread has no running loop, it is run with `asyncio.run`.- If it already has one (a caller inside an async function), it runs on a dedicated backgroundloop, so the calling thread can block without deadlocking. Pass `AsyncHook(hook, loop=...)` to- funnel onto a specific loop; it must not be running in the calling thread.+ funnel onto a specific loop; it must be running, and must not be the calling thread's own loop.+ Both are checked: a stopped loop and the caller's own loop each raise `ValueError` instead of+ blocking forever.A hook with no `async` methods is unaffected.diff --git a/docs/hooks/errors.md b/docs/hooks/errors.mdindex 7b7825e3..8dcde7be 100644--- a/docs/hooks/errors.md+++ b/docs/hooks/errors.md@@ -135,12 +135,19 @@ laya.load("convaiinnovations/laya", on_predict_end=metrics, hooks_timeout=2.0)```It can be set per instance or overridden per call on `predict_batch`, `system_one`,-`Router.route`, `Router.predict` and `ONNXAgent.system_one`.+`Router.route`, `Router.predict` and `ONNXAgent.system_one`. The value must be positive; `0` or a+negative number raises `ValueError` at the point it is set, rather than racing on a zero-length+`join`.++A timed hook runs on a worker thread in a copy of the caller's `contextvars` context, so a+request id or tracing span set by the caller is visible to the hook.One honest caveat: Python cannot interrupt a thread, so a timed-out hook keeps running in thebackground. The timeout bounds how long the request waits, not how long the hook lives. Use it tokeep a served request responsive, not to reclaim the work. For a hook that can hang, also give the-underlying call its own timeout (a socket or HTTP timeout).+underlying call its own timeout (a socket or HTTP timeout). Because the thread cannot be+reclaimed, a hook that hangs on every call grows threads one per call; give a hook that can hang+its own bound rather than relying on `hooks_timeout` to stop it.For an async hook, the coroutine runs on the event loop; a timeout on the calling side stillreturns after the limit, and the coroutine keeps running on the loop.diff --git a/laya/agent.py b/laya/agent.pyindex 31b9376a..7e779f03 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -27,7 +27,10 @@ from .common import (serialize_state,temp_bucket,)-from .hooks import HookRegistry, PredictContext, aggregate_usage, compose_hooks, dispatch, normalise_hooks+from .hooks import (+ HookRegistry, PredictContext, aggregate_usage, compose_hooks, dispatch, normalise_hooks,+ validate_timeout,+)def _fix_tokenizer_config(path: str):@@ -242,7 +245,7 @@ class Agent(HookRegistry):self.hooks = normalise_hooks(hooks, on_predict_start, on_predict_end)self.hooks_raise = bool(hooks_raise)self.hooks_concurrent = bool(hooks_concurrent)- self.hooks_timeout = None if hooks_timeout is None else float(hooks_timeout)+ self.hooks_timeout = None if hooks_timeout is None else validate_timeout(hooks_timeout)self._hooks_lock = threading.RLock() if not hooks_concurrent else Noneself._hooks_mutex = threading.Lock()self.model_id = model_id_or_path@@ -760,7 +763,7 @@ class Agent(HookRegistry):"""active = compose_hooks(self.hooks, hooks, on_predict_start, on_predict_end)raise_errors = self.hooks_raise if hooks_raise is None else bool(hooks_raise)- timeout = self.hooks_timeout if hooks_timeout is None else float(hooks_timeout)+ timeout = self.hooks_timeout if hooks_timeout is None else validate_timeout(hooks_timeout)ctx = PredictContext(states=states, questions=questions, model=self.model_id, agent=self,max_len=max_len, head_max_len=head_max_len)try:diff --git a/laya/hooks.py b/laya/hooks.pyindex bae6d80b..29a7b1a9 100644--- a/laya/hooks.py+++ b/laya/hooks.py@@ -7,6 +7,7 @@ Everything here is pure Python: importing `laya` must not start pulling torch.from __future__ import annotationsimport asyncio+import contextvarsimport inspectimport threadingimport time@@ -217,6 +218,21 @@ def compose_hooks(installed, hooks=None, on_predict_start=None, on_predict_end=Nreturn default_hooks() + list(installed) + normalise_hooks(hooks, on_predict_start, on_predict_end)+def validate_timeout(value: Optional[float]) -> Optional[float]:+ """Return *value* as a positive float, or ``None`` for no limit.++ A non-positive timeout is rejected here rather than left to+ ``thread.join``: ``join(0)`` and ``join(-1)`` return before the hook has+ started, so the outcome of a fast hook with such a value is a race.+ """+ if value is None:+ return None+ timeout = float(value)+ if timeout <= 0:+ raise ValueError("hooks_timeout must be a positive number or None; got %r" % (value,))+ return timeout++_BACKGROUND_LOOP: Optional[asyncio.AbstractEventLoop] = None_BACKGROUND_LOOP_LOCK = threading.Lock()@@ -239,10 +255,24 @@ def run_coroutine_sync(coro: Awaitable[Any], loop: Optional[asyncio.AbstractEvenUses `asyncio.run` when the calling thread has no running loop. When it does (a callerinside an async function, or a framework that already runs a loop), the coroutine is run ona dedicated background loop so the calling thread can block on it without deadlocking. Pass- `loop` to use a specific loop instead of the background one; it must not be running in the- calling thread.+ `loop` to use a specific loop instead of the background one.++ A supplied `loop` must be running somewhere, and must not be the calling thread's own+ loop. Both are checked: the first would otherwise block forever with no coroutine ever+ scheduled, and the second would block the only thread that could run the coroutine."""if loop is not None:+ if not loop.is_running():+ raise ValueError("run_coroutine_sync: the loop passed is not running")+ try:+ running = asyncio.get_running_loop()+ except RuntimeError:+ running = None+ if loop is running:+ raise ValueError(+ "run_coroutine_sync: the loop passed is running in the calling thread; "+ "blocking on it would deadlock"+ )return asyncio.run_coroutine_threadsafe(coro, loop).result()try:asyncio.get_running_loop()@@ -275,6 +305,11 @@ class AsyncHook:def __init__(self, hook: Any, loop: Optional[asyncio.AbstractEventLoop] = None):if isinstance(hook, type):raise TypeError("AsyncHook wraps an instance, not a class; got %s" % hook.__name__)+ if not any(hasattr(hook, event) for event in HOOK_EVENTS):+ raise TypeError(+ "AsyncHook wraps an object implementing at least one of %s; got %s"+ % (", ".join(HOOK_EVENTS), type(hook).__name__)+ )self.hook = hookself.loop = loop@@ -378,6 +413,8 @@ def _hook_name(method: Any) -> str:def _call_hook(method: Any, ctx: PredictContext, timeout: Optional[float]) -> None:"""Call one hook method, running its result if it is awaitable, under an optional timeout."""+ timeout = validate_timeout(timeout)+def invoke():result = method(ctx)if inspect.isawaitable(result):@@ -389,9 +426,14 @@ def _call_hook(method: Any, ctx: PredictContext, timeout: Optional[float]) -> Nobox: List[BaseException] = []+ # Run the hook in a copy of the caller's context, so a `contextvars` value (a request id,+ # a tracing span) set by the caller is visible to the hook even though it runs on another+ # thread. The no-timeout path runs inline and inherits the context already.+ context = contextvars.copy_context()+def runner():try:- invoke()+ context.run(invoke)except BaseException as exc: # noqa: BLE001 -- re-raised on the caller threadbox.append(exc)diff --git a/laya/onnx_agent.py b/laya/onnx_agent.pyindex 5622f19a..46df9b1d 100644--- a/laya/onnx_agent.py+++ b/laya/onnx_agent.py@@ -7,7 +7,10 @@ from typing import Any, Dict, Optional, Unionimport numpy as np-from laya.hooks import HookRegistry, PredictContext, aggregate_usage, compose_hooks, dispatch, normalise_hooks+from laya.hooks import (+ HookRegistry, PredictContext, aggregate_usage, compose_hooks, dispatch, normalise_hooks,+ validate_timeout,+)from laya.common import (QTYPES,answer_confidence,@@ -63,7 +66,7 @@ class ONNXAgent(HookRegistry):self.hooks = normalise_hooks(hooks, on_predict_start, on_predict_end)self.hooks_raise = bool(hooks_raise)self.hooks_concurrent = bool(hooks_concurrent)- self.hooks_timeout = None if hooks_timeout is None else float(hooks_timeout)+ self.hooks_timeout = None if hooks_timeout is None else validate_timeout(hooks_timeout)self._hooks_lock = threading.RLock() if not hooks_concurrent else Noneself._hooks_mutex = threading.Lock()self.model_id = model_id_or_path@@ -192,7 +195,7 @@ class ONNXAgent(HookRegistry):"""Evaluate typed questions, running any opt-in hooks around the inference."""active = compose_hooks(self.hooks, hooks, on_predict_start, on_predict_end)raise_errors = self.hooks_raise if hooks_raise is None else bool(hooks_raise)- timeout = self.hooks_timeout if hooks_timeout is None else float(hooks_timeout)+ timeout = self.hooks_timeout if hooks_timeout is None else validate_timeout(hooks_timeout)ctx = PredictContext(states=[state], questions=questions, model=self.model_id, agent=self,max_len=max_len, head_max_len=head_max_len)try:diff --git a/laya/router.py b/laya/router.pyindex c6fa2571..3adb3787 100644--- a/laya/router.py+++ b/laya/router.py@@ -35,7 +35,10 @@ import timefrom collections.abc import Sequence as SequenceABCfrom typing import Any, Dict, List, Optional, Sequence, Union-from .hooks import HookRegistry, PredictContext, aggregate_usage, compose_hooks, dispatch, normalise_hooks+from .hooks import (+ HookRegistry, PredictContext, aggregate_usage, compose_hooks, dispatch, normalise_hooks,+ validate_timeout,+)from .lang import analyse# The hub repo bundles all three checkpoints; only the requested subfolder is downloaded.@@ -231,7 +234,7 @@ class Router(HookRegistry):self.hooks = normalise_hooks(hooks, on_predict_start, on_predict_end)self.hooks_raise = bool(hooks_raise)self.hooks_concurrent = bool(hooks_concurrent)- self.hooks_timeout = None if hooks_timeout is None else float(hooks_timeout)+ self.hooks_timeout = None if hooks_timeout is None else validate_timeout(hooks_timeout)self._hooks_lock = threading.RLock() if not hooks_concurrent else Noneself._hooks_mutex = threading.Lock()self.models = dict(STANDALONE_MODELS if standalone_repos else DEFAULT_MODELS)@@ -419,7 +422,7 @@ class Router(HookRegistry):decision = self._route(state, questions, model=model, task=task, lang=lang, lang_guess=lang_guess)raise_errors = self.hooks_raise if hooks_raise is None else bool(hooks_raise)active = compose_hooks(self.hooks, hooks)- timeout = self.hooks_timeout if hooks_timeout is None else float(hooks_timeout)+ timeout = self.hooks_timeout if hooks_timeout is None else validate_timeout(hooks_timeout)ctx = PredictContext(states=[state], questions=questions or {}, decision=decision, router=self)dispatch(active, "on_route", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)return ctx.decision@@ -543,7 +546,7 @@ class Router(HookRegistry):"""active = compose_hooks(self.hooks, hooks, on_predict_start, on_predict_end)raise_errors = self.hooks_raise if hooks_raise is None else bool(hooks_raise)- timeout = self.hooks_timeout if hooks_timeout is None else float(hooks_timeout)+ timeout = self.hooks_timeout if hooks_timeout is None else validate_timeout(hooks_timeout)# Per-call hooks apply to the whole call, including on_route inside route().decision = self.route(state, questions, model=model, task=task, lang=lang,diff --git a/tests/test_hooks.py b/tests/test_hooks.pyindex 3ba09479..f1e290fc 100644--- a/tests/test_hooks.py+++ b/tests/test_hooks.py@@ -1117,6 +1117,59 @@ res = f.predict_batch(["s0"], QUESTIONS, on_predict_end=lambda ctx: None, hooks_check("timeout/fast hook unaffected", len(res), 1)+# --------------------------------------------------------------- input validation and context+import contextvars # noqa: E402++from laya.hooks import validate_timeout # noqa: E402++check_raises("timeout/zero is rejected", ValueError, lambda: validate_timeout(0))+check_raises("timeout/negative is rejected", ValueError, lambda: validate_timeout(-0.5))+check("timeout/positive passes through", validate_timeout(1.5), 1.5)+check("timeout/None means no limit", validate_timeout(None), None)++f = make_fake()+check_raises("timeout/zero per call is rejected", ValueError,+ lambda: f.predict_batch(["s0"], QUESTIONS, hooks_timeout=0))++not_running = asyncio.new_event_loop()+try:+ check_raises("async/a non-running loop is rejected", ValueError,+ lambda: run_coroutine_sync(_seven(), loop=not_running))+finally:+ not_running.close()+++async def _own_loop():+ own = asyncio.get_running_loop()+ try:+ run_coroutine_sync(_seven(), loop=own)+ except ValueError:+ return "raised"+ return "no"+++check("async/the calling thread's own loop is rejected", asyncio.run(_own_loop()), "raised")+check_raises("async/AsyncHook rejects an object with no events", TypeError,+ lambda: AsyncHook(object()))++_cv_seen = contextvars.ContextVar("cv_seen", default=None)+_cv_calls = []+++class CvHook:+ def on_predict_start(self, ctx):+ _cv_calls.append(_cv_seen.get())+++f = make_fake()+_token = _cv_seen.set("request-1")+try:+ f.predict_batch(["s0"], QUESTIONS, hooks=[CvHook()], hooks_timeout=1.0)+finally:+ _cv_seen.reset(_token)+check("timeout/a timed hook sees the caller's contextvars", _cv_calls, ["request-1"])++# --------------------------------------------------------------- reportprint("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f_ in FAIL: