Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(router): honour a request's token budget in predict_batch

Aashish5ae2f07ce0ed
diff --git a/README.md b/README.md
index 54c20cf9..14e775c2 100644
--- a/README.md
+++ b/README.md
@@ -343,9 +343,9 @@ requests = [
results = Router(max_loaded=1).predict_batch(requests)
# results stay in input order while compatible requests are batched by checkpoint
```
-Each item can independently set `model`, `task`, `lang`, or `lang_guess`. Use `route_batch(requests)` when you only want the ordered routing decisions without loading any checkpoint. `predict_many` is an alias for `predict_batch`.
+Each item can independently set `model`, `task`, `lang`, `lang_guess`, or the token budget (`max_len`, `head_max_len`). Use `route_batch(requests)` when you only want the ordered routing decisions without loading any checkpoint. `predict_many` is an alias for `predict_batch`.
-Requests are validated before model loading. Different requests may use different question schemas; requests sharing both a checkpoint and question schema are passed together to `Agent.predict_batch()`.
+Requests are validated before model loading. Different requests may use different question schemas; requests sharing a checkpoint, question schema and token budget are passed together to `Agent.predict_batch()`.
[Prediction hooks](#prediction-hooks) installed on the `Router` run once per request, as they do for `predict()`, so a redaction hook rewrites every state before the model sees it. Requests that share a checkpoint run all their start hooks before their shared forward pass; see [`docs/hooks/lifecycle.md`](docs/hooks/lifecycle.md#routerpredict_batch).
diff --git a/docs/hooks/lifecycle.md b/docs/hooks/lifecycle.md
index 76358b2c..b4209515 100644
--- a/docs/hooks/lifecycle.md
+++ b/docs/hooks/lifecycle.md
@@ -124,7 +124,9 @@ Router.predict_batch(requests, batch_size=...)
│
├─ load(checkpoint) ──► on_load / on_evict
├─ for each request of this checkpoint, in input order:
- │ ctx = PredictContext(states=[state], questions, decision, model, agent, router)
+ │ ctx = PredictContext(states=[state], questions, decision, model, agent, router,
+ │ max_len=request.get("max_len"),
+ │ head_max_len=request.get("head_max_len"))
│ on_predict_start a hook may redact, rewrite, set a token budget or skip
├─ group the requests left to infer by (questions, ctx.max_len, ctx.head_max_len)
│ agent.predict_batch(states, questions, ...) ──► one shared forward pass per group
@@ -143,6 +145,10 @@ Key points:
request only: requests are grouped for the forward pass after their start hooks have run.
Mutating a questions dict in place changes it for every request that shares that dict, and for
the caller, as it would with `predict`.
+- A request may carry its own `max_len` / `head_max_len`, the per-request form of the token budget
+ `predict` takes as call arguments. A start hook that sets `ctx.max_len` overrides it, because the
+ hook runs after the context is built. Requests that ask for different budgets cannot share a
+ forward pass, so a batch that mixes budgets makes one `agent.predict_batch` call per budget.
- Every started request gets exactly one `on_predict_end`, even when an earlier request's end hook
raises; the first such error is raised after all of them have run.
- If the batch fails, a request that did not get a result is reported as failed (`on_error`, with
diff --git a/laya/router.py b/laya/router.py
index 7dd999eb..bd1b5939 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -712,8 +712,14 @@ class Router(HookRegistry):
``Agent.predict_batch`` so their states can share forward passes. Results are
then restored to the original request order.
- Requests may independently specify ``model``, ``task``, ``lang`` or
- ``lang_guess`` and may use different question schemas.
+ Requests may independently specify ``model``, ``task``, ``lang``,
+ ``lang_guess``, ``max_len`` or ``head_max_len`` and may use different question schemas.
+ ``max_len`` / ``head_max_len`` are the per-request form of the token-budget override
+ ``predict`` takes as call arguments: they set the checkpoint's state and question-head
+ budgets for that one request, so a wide question can be asked without shrinking the
+ batch's other requests to the same window. Requests that ask for different budgets are
+ split into separate forward passes, since one ``Agent.predict_batch`` call carries one
+ budget for all its states. A start hook may still replace either value on ``ctx``.
Router-level predict hooks run per request, as ``predict`` runs them: each request
gets its own ``PredictContext``, so ``on_predict_start`` can replace that request's
@@ -726,7 +732,8 @@ class Router(HookRegistry):
Args:
requests: Sequence of request dictionaries. Every item requires ``state`` and
``questions`` and may include ``model``, ``task``, ``lang`` or
- ``lang_guess`` overrides.
+ ``lang_guess`` routing overrides and ``max_len`` / ``head_max_len`` token-budget
+ overrides.
batch_size: Optional maximum number of states per Agent forward-pass batch.
hooks_timeout: Override the Router's ``hooks_timeout`` for this call.
@@ -762,11 +769,15 @@ class Router(HookRegistry):
try:
# One context per request, built and started the way `predict` does it, so a
# start hook sees -- and can redact, rewrite or skip -- each request before it
- # joins a shared forward pass.
+ # joins a shared forward pass. A request's own `max_len` / `head_max_len` seed the
+ # context here; the grouping below already splits on those fields, so this is the
+ # only place the per-request budget can enter.
for i in indices:
ctx = PredictContext(states=[requests[i]["state"]], questions=requests[i]["questions"],
decision=dict(decisions[i]), model=model_name, agent=agent,
- router=self)
+ router=self,
+ max_len=requests[i].get("max_len"),
+ head_max_len=requests[i].get("head_max_len"))
started.append(ctx)
dispatch(active, "on_predict_start", ctx, raise_errors=raise_errors,
lock=self._hooks_lock, timeout=timeout)
diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py
index fa750e30..28e3296e 100644
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@ -90,6 +90,14 @@ for label, fn in (("Agent.predict_batch", Agent.predict_batch),
check_param(label, fn, "max_len", None)
check_param(label, fn, "head_max_len", None)
+# Router.predict_batch has no call-level budget: a heterogeneous batch sets it per request, and
+# the request keys `max_len` / `head_max_len` are read into each request's PredictContext.
+check_param("Router.predict_batch", Router.predict_batch, "batch_size", None)
+check_param("Router.predict_batch", Router.predict_batch, "hooks_timeout", None)
+for param in ("max_len", "head_max_len"):
+ check("Router.predict_batch/%s is per-request, not a call argument" % param,
+ param in sig(Router.predict_batch), False)
+
# route() takes per-call hooks so a hook can pin a checkpoint for one call
check_param("Router.route", Router.route, "hooks", None)
check_param("Router.route", Router.route, "hooks_raise", None)
diff --git a/tests/test_router_batch.py b/tests/test_router_batch.py
index 8677a9a6..6bb848af 100644
--- a/tests/test_router_batch.py
+++ b/tests/test_router_batch.py
@@ -306,3 +306,87 @@ def test_predict_and_predict_batch_pass_the_same_lang(monkeypatch):
calls.clear()
router.predict_batch([request("a", model="english", lang="de")])
assert calls[-1]["lang"] == via_predict == "de"
+
+
+def _budget_recording_router(monkeypatch, hooks=(), agent=None):
+ """A Router over a fake agent that records the token budget each forward pass received."""
+ import laya.agent
+
+ calls = []
+
+ class Recording:
+ def __init__(self, repo, *, device, token, subfolder):
+ self.checkpoint = subfolder or "english"
+
+ def predict_batch(self, states, questions, batch_size=None, **overrides):
+ calls.append({"n": len(states), "max_len": overrides.get("max_len"),
+ "head_max_len": overrides.get("head_max_len")})
+ return [{"model": "fake", "answers": {}, "usage": {}} for _ in states]
+
+ def system_one(self, state, questions, **overrides):
+ calls.append({"n": 1, "max_len": overrides.get("max_len"),
+ "head_max_len": overrides.get("head_max_len")})
+ return {"model": "fake", "answers": {}, "usage": {}}
+
+ monkeypatch.setattr(laya.agent, "Agent", agent or Recording)
+ return Router(max_loaded=1, default="english", hooks=list(hooks)), calls
+
+
+def test_request_token_budget_reaches_the_forward_pass(monkeypatch):
+ """A request's `max_len` / `head_max_len` are read into its context, which is where the
+ grouping below already keys on them. Two requests that ask for the same wide budget share a
+ forward pass; the one that asks for nothing is grouped apart, because the budgets differ."""
+ router, calls = _budget_recording_router(monkeypatch)
+ router.predict_batch([request("a", max_len=1024, head_max_len=512),
+ request("b", max_len=1024, head_max_len=512),
+ request("c")])
+ assert calls == [{"n": 2, "max_len": 1024, "head_max_len": 512},
+ {"n": 1, "max_len": None, "head_max_len": None}]
+
+
+def test_predict_and_predict_batch_pass_the_same_budget(monkeypatch):
+ """The invariant: one request, either entry point, the same token budget."""
+ router, calls = _budget_recording_router(monkeypatch)
+ router.predict("a", Q, model="english", max_len=1024, head_max_len=512)
+ via_predict = calls[-1]
+ calls.clear()
+ router.predict_batch([request("a", model="english", max_len=1024, head_max_len=512)])
+ assert calls[-1] == dict(via_predict, n=1)
+
+
+def test_requests_without_a_budget_forward_no_budget_keys(monkeypatch):
+ """`predict` passes the budget only when set, so an Agent-like object whose methods predate
+ the arguments still works. Naming the keys `None` must behave like leaving them out."""
+ class NoBudgetArgs:
+ def __init__(self, repo, *, device, token, subfolder):
+ pass
+
+ def predict_batch(self, states, questions, batch_size=None):
+ return [{"model": "fake", "answers": {}, "usage": {}} for _ in states]
+
+ router, _ = _budget_recording_router(monkeypatch, agent=NoBudgetArgs)
+ out = router.predict_batch([request("a"), request("b", max_len=None, head_max_len=None)])
+ assert len(out) == 2
+
+
+def test_start_hook_budget_outranks_the_request_budget(monkeypatch):
+ """The context is seeded from the request and only then started, so a hook that sets a budget
+ wins -- the same ordering `predict` has."""
+ class Narrow:
+ def on_predict_start(self, ctx):
+ ctx.max_len = 64
+
+ router, calls = _budget_recording_router(monkeypatch, hooks=[Narrow()])
+ router.predict_batch([request("a", max_len=1024)])
+ assert calls == [{"n": 1, "max_len": 64, "head_max_len": None}]
+
+
+def test_request_budget_does_not_change_routing(monkeypatch):
+ """The budget is an inference argument: the routed checkpoint and `routing` must be what the
+ same request gets without it."""
+ router, calls = _budget_recording_router(monkeypatch)
+ plain = router.predict_batch([request("a")])
+ calls.clear()
+ wide = router.predict_batch([request("a", max_len=1024, head_max_len=512)])
+ assert [r["routing"] for r in wide] == [r["routing"] for r in plain]
+