Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(router): forward predict_batch sort_by_length to every grouped Agent call

Aashishab3e7c5f86d9
diff --git a/README.md b/README.md
index 54c20cf9..2ebecd06 100644
--- a/README.md
+++ b/README.md
@@ -349,9 +349,10 @@ Requests are validated before model loading. Different requests may use differen
[Prediction hooks](#prediction-hooks) installed on the `Router` run once per request, as they do for `predict()`, so a redaction hook rewrites every state before the model sees it. Requests that share a checkpoint run all their start hooks before their shared forward pass; see [`docs/hooks/lifecycle.md`](docs/hooks/lifecycle.md#routerpredict_batch).
-You can also bound the Agent-level forward-pass batch size:
+You can also bound the Agent-level forward-pass batch size, and forward the length grouping knob
+to every group (see [Batch Mode](#batch-mode-score-many-states-in-one-forward-pass)):
```python
-results = router.predict_batch(requests, batch_size=8)
+results = router.predict_batch(requests, batch_size=8, sort_by_length=True)
```
### Why Route: The Evidence
diff --git a/laya/router.py b/laya/router.py
index 7dd999eb..3d1b8b10 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -704,6 +704,7 @@ class Router(HookRegistry):
requests: Sequence[Dict[str, Any]],
batch_size: Optional[int] = None,
hooks_timeout: Optional[float] = None,
+ sort_by_length: bool = False,
) -> List[Dict[str, Any]]:
"""Route and execute a heterogeneous request batch with minimal model churn.
@@ -729,6 +730,10 @@ class Router(HookRegistry):
``lang_guess`` overrides.
batch_size: Optional maximum number of states per Agent forward-pass batch.
hooks_timeout: Override the Router's ``hooks_timeout`` for this call.
+ sort_by_length: Forwarded to every ``Agent.predict_batch`` call, so each question
+ group pads to a shorter maximum; see ``Agent.predict_batch``. Results retain the
+ input order either way. Silently dropped for an attached agent whose
+ ``predict_batch`` predates the knob (#294).
Returns:
One normal Router prediction result per request, in the same order as the input.
@@ -822,6 +827,10 @@ class Router(HookRegistry):
batch_kwargs = dict(group["overrides"])
if group["lang"] is not None:
batch_kwargs["lang"] = group["lang"]
+ if sort_by_length:
+ # Passed only when on, as the token-budget overrides are: agents attached
+ # via `attach()` may predate the knob (#294).
+ batch_kwargs["sort_by_length"] = True
skip = _SKIP_DEFAULTS.set(True)
try:
batch_results = agent.predict_batch(
@@ -832,9 +841,14 @@ class Router(HookRegistry):
)
except TypeError as e:
# Same tolerance `predict` has for an Agent-like object whose
- # `predict_batch` predates the `lang` argument.
- if batch_kwargs.get("lang") is not None and "unexpected keyword argument 'lang'" in str(e):
- batch_kwargs.pop("lang")
+ # `predict_batch` predates the `lang` (or `sort_by_length`) argument.
+ retried = False
+ for kwarg in ("lang", "sort_by_length"):
+ if batch_kwargs.get(kwarg) is not None and \
+ "unexpected keyword argument '%s'" % kwarg in str(e):
+ batch_kwargs.pop(kwarg)
+ retried = True
+ if retried:
batch_results = agent.predict_batch(
[ctx.states[0] for _, ctx in items],
group["questions"],
diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py
index fa750e30..e2646673 100644
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@ -81,6 +81,9 @@ for label, fn in (("Agent.predict_batch", Agent.predict_batch),
check_param(label, fn, "hooks_timeout", None)
check_param("Agent.predict_batch", Agent.predict_batch, "batch_size", None)
+check_param("Agent.predict_batch", Agent.predict_batch, "sort_by_length", False)
+check_param("Router.predict_batch", Router.predict_batch, "batch_size", None)
+check_param("Router.predict_batch", Router.predict_batch, "sort_by_length", False)
# per-call token-budget overrides
for label, fn in (("Agent.predict_batch", Agent.predict_batch),
diff --git a/tests/test_router_batch.py b/tests/test_router_batch.py
index 8677a9a6..33815034 100644
--- a/tests/test_router_batch.py
+++ b/tests/test_router_batch.py
@@ -306,3 +306,56 @@ def test_predict_and_predict_batch_pass_the_same_lang(monkeypatch):
calls.clear()
router.predict_batch([request("a", model="english", lang="de")])
assert calls[-1]["lang"] == via_predict == "de"
+
+
+def _sort_recording_router(monkeypatch, accepts_sort):
+ """A Router whose attached agent records the sort_by_length it was called with."""
+ import laya.agent
+
+ calls = []
+
+ class Agent:
+ def __init__(self, repo, *, device, token, subfolder):
+ pass
+
+ if accepts_sort:
+ def predict_batch(self, states, questions, batch_size=None, sort_by_length=False,
+ **overrides):
+ calls.append({"n": len(states), "sort": sort_by_length})
+ return [{"model": "fake", "answers": {}, "usage": {}} for _ in states]
+ else:
+ # Strict signature, no **overrides: passing sort_by_length must raise TypeError,
+ # exactly like a real Agent that predates #294.
+ def predict_batch(self, states, questions, batch_size=None):
+ calls.append({"n": len(states), "sort": "unsupported"})
+ return [{"model": "fake", "answers": {}, "usage": {}} for _ in states]
+
+ def system_one(self, state, questions):
+ return self.predict_batch([state], questions)[0]
+
+ monkeypatch.setattr(laya.agent, "Agent", Agent)
+ return Router(max_loaded=1, default="english"), calls
+
+
+def test_sort_by_length_reaches_every_agent_call(monkeypatch):
+ """`Agent.predict_batch` has had `sort_by_length` since #294, but `Router.predict_batch`
+ never forwarded it, so routing -- the normal entry point -- silently lost length grouping.
+ Every model/question-schema group the batch splits into must get the knob."""
+ router, calls = _sort_recording_router(monkeypatch, accepts_sort=True)
+ two_schemas = {"intent": {"type": "noul", "instructions": "Urgent?"}} # different schema text -> own group
+ router.predict_batch([request("a"), request("b"),
+ {"state": "c", "questions": two_schemas}], sort_by_length=True)
+ assert [c["sort"] for c in calls] == [True, True] # one call per group, each with the knob
+ calls.clear()
+ router.predict_batch([request("a"), request("b")]) # default stays off
+ assert [c["sort"] for c in calls] == [False]
+
+
+def test_sort_by_length_is_dropped_for_agents_that_predate_it(monkeypatch):
+ """An attached agent-like object whose `predict_batch` predates #294 must keep serving the
+ batch instead of raising TypeError -- the same tolerance the `lang` forwarding has."""
+ router, calls = _sort_recording_router(monkeypatch, accepts_sort=False)
+ results = router.predict_batch([request("a"), request("b")], sort_by_length=True)
+ assert len(results) == 2
+ assert [c["sort"] for c in calls] == ["unsupported"] # one group, retry without the knob
+ assert [r["routing"]["model"] for r in results] == ["english", "english"]