Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(hooks): honour hooks_timeout on Router.predict_batch
Emiliano Gandini Outeda045d5cced573
diff --git a/docs/hooks/errors.md b/docs/hooks/errors.mdindex 8dcde7be..4782077c 100644--- a/docs/hooks/errors.md+++ b/docs/hooks/errors.md@@ -152,6 +152,11 @@ its own bound rather than relying on `hooks_timeout` to stop it.For an async hook, the coroutine runs on the event loop; a timeout on the calling side stillreturns after the limit, and the coroutine keeps running on the loop.+The timeout also releases the `hooks_concurrent=False` lock: dispatch waits for the hook only up+to the limit, then moves on, while the timed-out hook keeps running outside the lock. So the lock+serialises the hooks that finish in time, not every hook that was ever started; a hook that+overruns no longer blocks the ones behind it.+## Choosing a policy| hook kind | recommended | why |diff --git a/laya/router.py b/laya/router.pyindex 3adb3787..a70e46dd 100644--- a/laya/router.py+++ b/laya/router.py@@ -673,6 +673,7 @@ class Router(HookRegistry):self,requests: Sequence[Dict[str, Any]],batch_size: Optional[int] = None,+ hooks_timeout: Optional[float] = None,) -> List[Dict[str, Any]]:"""Route and execute a heterogeneous request batch with minimal model churn.@@ -697,6 +698,7 @@ class Router(HookRegistry):``questions`` and may include ``model``, ``task``, ``lang`` or``lang_guess`` overrides.batch_size: Optional maximum number of states per Agent forward-pass batch.+ hooks_timeout: Override the Router's ``hooks_timeout`` for this call.Returns:One normal Router prediction result per request, in the same order as the input.@@ -722,6 +724,7 @@ class Router(HookRegistry):# for a request that arrived through `predict_batch`.active = compose_hooks(self.hooks)raise_errors = self.hooks_raise+ timeout = self.hooks_timeout if hooks_timeout is None else validate_timeout(hooks_timeout)for model_name, indices in groups.items():agent = self.load(model_name)@@ -735,7 +738,8 @@ class Router(HookRegistry):decision=dict(decisions[i]), model=model_name, agent=agent,router=self)started.append(ctx)- dispatch(active, "on_predict_start", ctx, raise_errors=raise_errors, lock=self._hooks_lock)+ dispatch(active, "on_predict_start", ctx, raise_errors=raise_errors,+ lock=self._hooks_lock, timeout=timeout)# Agent.predict_batch evaluates one shared question schema and token budget over# many states. Preserve Router's heterogeneous-request API by splitting each@@ -826,16 +830,17 @@ class Router(HookRegistry):if ctx.results is None:ctx.error = exctry:- dispatch(active, "on_error", ctx, raise_errors=raise_errors, lock=self._hooks_lock)+ dispatch(active, "on_error", ctx, raise_errors=raise_errors,+ lock=self._hooks_lock, timeout=timeout)except BaseException as hook_exc:exc.__context__ = hook_exctry:- self._end_contexts(active, started, raise_errors)+ self._end_contexts(active, started, raise_errors, timeout)except BaseException as hook_exc:exc.__context__ = hook_excraise- self._end_contexts(active, started, raise_errors)+ self._end_contexts(active, started, raise_errors, timeout)for i, ctx in zip(indices, started):results[i] = ctx.results[0]@@ -848,7 +853,8 @@ class Router(HookRegistry):predict_many = predict_batch- def _end_contexts(self, active: List[Any], contexts: List[PredictContext], raise_errors: bool) -> None:+ def _end_contexts(self, active: List[Any], contexts: List[PredictContext], raise_errors: bool,+ timeout: Optional[float] = None) -> None:"""Finish each request of a batch the way `predict`'s `finally` finishes one.Every context gets its `on_predict_end` even if an earlier one's end hook raises; the@@ -864,7 +870,8 @@ class Router(HookRegistry):first_error: Optional[BaseException] = Nonefor ctx in contexts:try:- dispatch(active, "on_predict_end", ctx, raise_errors=raise_errors, lock=self._hooks_lock)+ dispatch(active, "on_predict_end", ctx, raise_errors=raise_errors,+ lock=self._hooks_lock, timeout=timeout)except BaseException as hook_exc:if ctx.error is not None:ctx.error.__context__ = hook_excdiff --git a/tests/test_router_batch.py b/tests/test_router_batch.pyindex bfdb51a5..8677a9a6 100644--- a/tests/test_router_batch.py+++ b/tests/test_router_batch.py@@ -167,6 +167,20 @@ def test_same_checkpoint_different_questions_split_agent_batches(fake_agent):assert calls[1][2] == q2+def test_predict_batch_honours_hooks_timeout(fake_agent):+ def slow_hook(ctx):+ sleep(0.3)++ router = Router(hooks_timeout=0.05, on_predict_start=slow_hook)+ with pytest.raises(TimeoutError):+ router.predict_batch([request("one")])++ # A per-call override wins, exactly as it does on `predict`.+ router = Router(hooks_timeout=0.05, on_predict_start=slow_hook)+ results = router.predict_batch([request("one")], hooks_timeout=5.0)+ assert len(results) == 1++def test_route_batch_forwards_lang_guess(fake_agent):built, _ = fake_agentrouter = Router()