Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(hooks): honour hooks_timeout on Router.predict_batch

Emiliano Gandini Outeda045d5cced573
diff --git a/docs/hooks/errors.md b/docs/hooks/errors.md
index 8dcde7be..4782077c 100644
--- a/docs/hooks/errors.md
+++ b/docs/hooks/errors.md
@@ -152,6 +152,11 @@ its own bound rather than relying on `hooks_timeout` to stop it.
For an async hook, the coroutine runs on the event loop; a timeout on the calling side still
returns after the limit, and the coroutine keeps running on the loop.
+The timeout also releases the `hooks_concurrent=False` lock: dispatch waits for the hook only up
+to the limit, then moves on, while the timed-out hook keeps running outside the lock. So the lock
+serialises the hooks that finish in time, not every hook that was ever started; a hook that
+overruns no longer blocks the ones behind it.
+
## Choosing a policy
| hook kind | recommended | why |
diff --git a/laya/router.py b/laya/router.py
index 3adb3787..a70e46dd 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -673,6 +673,7 @@ class Router(HookRegistry):
self,
requests: Sequence[Dict[str, Any]],
batch_size: Optional[int] = None,
+ hooks_timeout: Optional[float] = None,
) -> List[Dict[str, Any]]:
"""Route and execute a heterogeneous request batch with minimal model churn.
@@ -697,6 +698,7 @@ class Router(HookRegistry):
``questions`` and may include ``model``, ``task``, ``lang`` or
``lang_guess`` overrides.
batch_size: Optional maximum number of states per Agent forward-pass batch.
+ hooks_timeout: Override the Router's ``hooks_timeout`` for this call.
Returns:
One normal Router prediction result per request, in the same order as the input.
@@ -722,6 +724,7 @@ class Router(HookRegistry):
# for a request that arrived through `predict_batch`.
active = compose_hooks(self.hooks)
raise_errors = self.hooks_raise
+ timeout = self.hooks_timeout if hooks_timeout is None else validate_timeout(hooks_timeout)
for model_name, indices in groups.items():
agent = self.load(model_name)
@@ -735,7 +738,8 @@ class Router(HookRegistry):
decision=dict(decisions[i]), model=model_name, agent=agent,
router=self)
started.append(ctx)
- dispatch(active, "on_predict_start", ctx, raise_errors=raise_errors, lock=self._hooks_lock)
+ dispatch(active, "on_predict_start", ctx, raise_errors=raise_errors,
+ lock=self._hooks_lock, timeout=timeout)
# Agent.predict_batch evaluates one shared question schema and token budget over
# many states. Preserve Router's heterogeneous-request API by splitting each
@@ -826,16 +830,17 @@ class Router(HookRegistry):
if ctx.results is None:
ctx.error = exc
try:
- dispatch(active, "on_error", ctx, raise_errors=raise_errors, lock=self._hooks_lock)
+ dispatch(active, "on_error", ctx, raise_errors=raise_errors,
+ lock=self._hooks_lock, timeout=timeout)
except BaseException as hook_exc:
exc.__context__ = hook_exc
try:
- self._end_contexts(active, started, raise_errors)
+ self._end_contexts(active, started, raise_errors, timeout)
except BaseException as hook_exc:
exc.__context__ = hook_exc
raise
- self._end_contexts(active, started, raise_errors)
+ self._end_contexts(active, started, raise_errors, timeout)
for i, ctx in zip(indices, started):
results[i] = ctx.results[0]
@@ -848,7 +853,8 @@ class Router(HookRegistry):
predict_many = predict_batch
- def _end_contexts(self, active: List[Any], contexts: List[PredictContext], raise_errors: bool) -> None:
+ def _end_contexts(self, active: List[Any], contexts: List[PredictContext], raise_errors: bool,
+ timeout: Optional[float] = None) -> None:
"""Finish each request of a batch the way `predict`'s `finally` finishes one.
Every context gets its `on_predict_end` even if an earlier one's end hook raises; the
@@ -864,7 +870,8 @@ class Router(HookRegistry):
first_error: Optional[BaseException] = None
for ctx in contexts:
try:
- dispatch(active, "on_predict_end", ctx, raise_errors=raise_errors, lock=self._hooks_lock)
+ dispatch(active, "on_predict_end", ctx, raise_errors=raise_errors,
+ lock=self._hooks_lock, timeout=timeout)
except BaseException as hook_exc:
if ctx.error is not None:
ctx.error.__context__ = hook_exc
diff --git a/tests/test_router_batch.py b/tests/test_router_batch.py
index bfdb51a5..8677a9a6 100644
--- a/tests/test_router_batch.py
+++ b/tests/test_router_batch.py
@@ -167,6 +167,20 @@ def test_same_checkpoint_different_questions_split_agent_batches(fake_agent):
assert calls[1][2] == q2
+def test_predict_batch_honours_hooks_timeout(fake_agent):
+ def slow_hook(ctx):
+ sleep(0.3)
+
+ router = Router(hooks_timeout=0.05, on_predict_start=slow_hook)
+ with pytest.raises(TimeoutError):
+ router.predict_batch([request("one")])
+
+ # A per-call override wins, exactly as it does on `predict`.
+ router = Router(hooks_timeout=0.05, on_predict_start=slow_hook)
+ results = router.predict_batch([request("one")], hooks_timeout=5.0)
+ assert len(results) == 1
+
+
def test_route_batch_forwards_lang_guess(fake_agent):
built, _ = fake_agent
router = Router()