Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(router): forward predict_batch sort_by_length to every grouped Agent call
Aashishab3e7c5f86d9
diff --git a/README.md b/README.mdindex 54c20cf9..2ebecd06 100644--- a/README.md+++ b/README.md@@ -349,9 +349,10 @@ Requests are validated before model loading. Different requests may use differen[Prediction hooks](#prediction-hooks) installed on the `Router` run once per request, as they do for `predict()`, so a redaction hook rewrites every state before the model sees it. Requests that share a checkpoint run all their start hooks before their shared forward pass; see [`docs/hooks/lifecycle.md`](docs/hooks/lifecycle.md#routerpredict_batch).-You can also bound the Agent-level forward-pass batch size:+You can also bound the Agent-level forward-pass batch size, and forward the length grouping knob+to every group (see [Batch Mode](#batch-mode-score-many-states-in-one-forward-pass)):```python-results = router.predict_batch(requests, batch_size=8)+results = router.predict_batch(requests, batch_size=8, sort_by_length=True)```### Why Route: The Evidencediff --git a/laya/router.py b/laya/router.pyindex 7dd999eb..3d1b8b10 100644--- a/laya/router.py+++ b/laya/router.py@@ -704,6 +704,7 @@ class Router(HookRegistry):requests: Sequence[Dict[str, Any]],batch_size: Optional[int] = None,hooks_timeout: Optional[float] = None,+ sort_by_length: bool = False,) -> List[Dict[str, Any]]:"""Route and execute a heterogeneous request batch with minimal model churn.@@ -729,6 +730,10 @@ class Router(HookRegistry):``lang_guess`` overrides.batch_size: Optional maximum number of states per Agent forward-pass batch.hooks_timeout: Override the Router's ``hooks_timeout`` for this call.+ sort_by_length: Forwarded to every ``Agent.predict_batch`` call, so each question+ group pads to a shorter maximum; see ``Agent.predict_batch``. Results retain the+ input order either way. Silently dropped for an attached agent whose+ ``predict_batch`` predates the knob (#294).Returns:One normal Router prediction result per request, in the same order as the input.@@ -822,6 +827,10 @@ class Router(HookRegistry):batch_kwargs = dict(group["overrides"])if group["lang"] is not None:batch_kwargs["lang"] = group["lang"]+ if sort_by_length:+ # Passed only when on, as the token-budget overrides are: agents attached+ # via `attach()` may predate the knob (#294).+ batch_kwargs["sort_by_length"] = Trueskip = _SKIP_DEFAULTS.set(True)try:batch_results = agent.predict_batch(@@ -832,9 +841,14 @@ class Router(HookRegistry):)except TypeError as e:# Same tolerance `predict` has for an Agent-like object whose- # `predict_batch` predates the `lang` argument.- if batch_kwargs.get("lang") is not None and "unexpected keyword argument 'lang'" in str(e):- batch_kwargs.pop("lang")+ # `predict_batch` predates the `lang` (or `sort_by_length`) argument.+ retried = False+ for kwarg in ("lang", "sort_by_length"):+ if batch_kwargs.get(kwarg) is not None and \+ "unexpected keyword argument '%s'" % kwarg in str(e):+ batch_kwargs.pop(kwarg)+ retried = True+ if retried:batch_results = agent.predict_batch([ctx.states[0] for _, ctx in items],group["questions"],diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.pyindex fa750e30..e2646673 100644--- a/tests/test_hooks_api.py+++ b/tests/test_hooks_api.py@@ -81,6 +81,9 @@ for label, fn in (("Agent.predict_batch", Agent.predict_batch),check_param(label, fn, "hooks_timeout", None)check_param("Agent.predict_batch", Agent.predict_batch, "batch_size", None)+check_param("Agent.predict_batch", Agent.predict_batch, "sort_by_length", False)+check_param("Router.predict_batch", Router.predict_batch, "batch_size", None)+check_param("Router.predict_batch", Router.predict_batch, "sort_by_length", False)# per-call token-budget overridesfor label, fn in (("Agent.predict_batch", Agent.predict_batch),diff --git a/tests/test_router_batch.py b/tests/test_router_batch.pyindex 8677a9a6..33815034 100644--- a/tests/test_router_batch.py+++ b/tests/test_router_batch.py@@ -306,3 +306,56 @@ def test_predict_and_predict_batch_pass_the_same_lang(monkeypatch):calls.clear()router.predict_batch([request("a", model="english", lang="de")])assert calls[-1]["lang"] == via_predict == "de"+++def _sort_recording_router(monkeypatch, accepts_sort):+ """A Router whose attached agent records the sort_by_length it was called with."""+ import laya.agent++ calls = []++ class Agent:+ def __init__(self, repo, *, device, token, subfolder):+ pass++ if accepts_sort:+ def predict_batch(self, states, questions, batch_size=None, sort_by_length=False,+ **overrides):+ calls.append({"n": len(states), "sort": sort_by_length})+ return [{"model": "fake", "answers": {}, "usage": {}} for _ in states]+ else:+ # Strict signature, no **overrides: passing sort_by_length must raise TypeError,+ # exactly like a real Agent that predates #294.+ def predict_batch(self, states, questions, batch_size=None):+ calls.append({"n": len(states), "sort": "unsupported"})+ return [{"model": "fake", "answers": {}, "usage": {}} for _ in states]++ def system_one(self, state, questions):+ return self.predict_batch([state], questions)[0]++ monkeypatch.setattr(laya.agent, "Agent", Agent)+ return Router(max_loaded=1, default="english"), calls+++def test_sort_by_length_reaches_every_agent_call(monkeypatch):+ """`Agent.predict_batch` has had `sort_by_length` since #294, but `Router.predict_batch`+ never forwarded it, so routing -- the normal entry point -- silently lost length grouping.+ Every model/question-schema group the batch splits into must get the knob."""+ router, calls = _sort_recording_router(monkeypatch, accepts_sort=True)+ two_schemas = {"intent": {"type": "noul", "instructions": "Urgent?"}} # different schema text -> own group+ router.predict_batch([request("a"), request("b"),+ {"state": "c", "questions": two_schemas}], sort_by_length=True)+ assert [c["sort"] for c in calls] == [True, True] # one call per group, each with the knob+ calls.clear()+ router.predict_batch([request("a"), request("b")]) # default stays off+ assert [c["sort"] for c in calls] == [False]+++def test_sort_by_length_is_dropped_for_agents_that_predate_it(monkeypatch):+ """An attached agent-like object whose `predict_batch` predates #294 must keep serving the+ batch instead of raising TypeError -- the same tolerance the `lang` forwarding has."""+ router, calls = _sort_recording_router(monkeypatch, accepts_sort=False)+ results = router.predict_batch([request("a"), request("b")], sort_by_length=True)+ assert len(results) == 2+ assert [c["sort"] for c in calls] == ["unsupported"] # one group, retry without the knob+ assert [r["routing"]["model"] for r in results] == ["english", "english"]