Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #583 from aashish254/feat-router-batch-budget

Nandakishora7f7b4bd4e37
diff --cc laya/router.py
index 3da10d34,bd1b5939..f673b5a9
--- a/laya/router.py
+++ b/laya/router.py
@@@ -979,13 -732,10 +985,14 @@@ class Router(HookRegistry)
Args:
requests: Sequence of request dictionaries. Every item requires ``state`` and
``questions`` and may include ``model``, ``task``, ``lang`` or
- ``lang_guess`` overrides.
+ ``lang_guess`` routing overrides and ``max_len`` / ``head_max_len`` token-budget
+ overrides.
batch_size: Optional maximum number of states per Agent forward-pass batch.
hooks_timeout: Override the Router's ``hooks_timeout`` for this call.
+ sort_by_length: Forwarded to every ``Agent.predict_batch`` call, so each question
+ group pads to a shorter maximum; see ``Agent.predict_batch``. Results retain the
+ input order either way. Silently dropped for an attached agent whose
+ ``predict_batch`` predates the knob (#294).
Returns:
One normal Router prediction result per request, in the same order as the input.
diff --cc tests/test_router_batch.py
index 1fbd008a,6bb848af..cfdcaae3
--- a/tests/test_router_batch.py
+++ b/tests/test_router_batch.py
@@@ -308,137 -308,85 +308,221 @@@ def test_predict_and_predict_batch_pass
assert calls[-1]["lang"] == via_predict == "de"
+ def _budget_recording_router(monkeypatch, hooks=(), agent=None):
+ """A Router over a fake agent that records the token budget each forward pass received."""
+ import laya.agent
+
+ calls = []
+
+ class Recording:
+ def __init__(self, repo, *, device, token, subfolder):
+ self.checkpoint = subfolder or "english"
+
+ def predict_batch(self, states, questions, batch_size=None, **overrides):
+ calls.append({"n": len(states), "max_len": overrides.get("max_len"),
+ "head_max_len": overrides.get("head_max_len")})
+ return [{"model": "fake", "answers": {}, "usage": {}} for _ in states]
+
+ def system_one(self, state, questions, **overrides):
+ calls.append({"n": 1, "max_len": overrides.get("max_len"),
+ "head_max_len": overrides.get("head_max_len")})
+ return {"model": "fake", "answers": {}, "usage": {}}
+
+ monkeypatch.setattr(laya.agent, "Agent", agent or Recording)
+ return Router(max_loaded=1, default="english", hooks=list(hooks)), calls
+
+
+ def test_request_token_budget_reaches_the_forward_pass(monkeypatch):
+ """A request's `max_len` / `head_max_len` are read into its context, which is where the
+ grouping below already keys on them. Two requests that ask for the same wide budget share a
+ forward pass; the one that asks for nothing is grouped apart, because the budgets differ."""
+ router, calls = _budget_recording_router(monkeypatch)
+ router.predict_batch([request("a", max_len=1024, head_max_len=512),
+ request("b", max_len=1024, head_max_len=512),
+ request("c")])
+ assert calls == [{"n": 2, "max_len": 1024, "head_max_len": 512},
+ {"n": 1, "max_len": None, "head_max_len": None}]
+
+
+ def test_predict_and_predict_batch_pass_the_same_budget(monkeypatch):
+ """The invariant: one request, either entry point, the same token budget."""
+ router, calls = _budget_recording_router(monkeypatch)
+ router.predict("a", Q, model="english", max_len=1024, head_max_len=512)
+ via_predict = calls[-1]
+ calls.clear()
+ router.predict_batch([request("a", model="english", max_len=1024, head_max_len=512)])
+ assert calls[-1] == dict(via_predict, n=1)
+
+
+ def test_requests_without_a_budget_forward_no_budget_keys(monkeypatch):
+ """`predict` passes the budget only when set, so an Agent-like object whose methods predate
+ the arguments still works. Naming the keys `None` must behave like leaving them out."""
+ class NoBudgetArgs:
+ def __init__(self, repo, *, device, token, subfolder):
+ pass
+
+ def predict_batch(self, states, questions, batch_size=None):
+ return [{"model": "fake", "answers": {}, "usage": {}} for _ in states]
+
+ router, _ = _budget_recording_router(monkeypatch, agent=NoBudgetArgs)
+ out = router.predict_batch([request("a"), request("b", max_len=None, head_max_len=None)])
+ assert len(out) == 2
+
+
+ def test_start_hook_budget_outranks_the_request_budget(monkeypatch):
+ """The context is seeded from the request and only then started, so a hook that sets a budget
+ wins -- the same ordering `predict` has."""
+ class Narrow:
+ def on_predict_start(self, ctx):
+ ctx.max_len = 64
+
+ router, calls = _budget_recording_router(monkeypatch, hooks=[Narrow()])
+ router.predict_batch([request("a", max_len=1024)])
+ assert calls == [{"n": 1, "max_len": 64, "head_max_len": None}]
+
+
+ def test_request_budget_does_not_change_routing(monkeypatch):
+ """The budget is an inference argument: the routed checkpoint and `routing` must be what the
+ same request gets without it."""
+ router, calls = _budget_recording_router(monkeypatch)
+ plain = router.predict_batch([request("a")])
+ calls.clear()
+ wide = router.predict_batch([request("a", max_len=1024, head_max_len=512)])
+ assert [r["routing"] for r in wide] == [r["routing"] for r in plain]
+
++
++
+def test_router_predict_and_predict_batch_min_confidence(monkeypatch):
+ """`min_confidence` gates answers below threshold on both predict and predict_batch (#361)."""
+ import laya.agent
+
+ class Agent:
+ def __init__(self, repo, *, device, token, subfolder):
+ pass
+
+ def system_one(self, state, questions, **overrides):
+ return {
+ "model": "fake",
+ "answers": {
+ "q1": {"type": "choice", "choice": "yes", "answer_confidence": 0.95},
+ "q2": {"type": "choice", "choice": "no", "answer_confidence": 0.60},
+ },
+ }
+
+ def predict_batch(self, states, questions, batch_size=None, **overrides):
+ return [self.system_one(s, questions, **overrides) for s in states]
+
+ monkeypatch.setattr(laya.agent, "Agent", Agent)
+ router = Router(max_loaded=1, default="english")
+
+ # Default min_confidence=None: no low_confidence flags
+ res_default = router.predict("hello", {"q1": {}}, model="english")
+ assert "low_confidence" not in res_default["answers"]["q1"]
+ assert "low_confidence" not in res_default["answers"]["q2"]
+
+ # min_confidence=0.80 on predict: q2 flagged, q1 unflagged
+ res_gated = router.predict("hello", {"q1": {}}, model="english", min_confidence=0.80)
+ assert "low_confidence" not in res_gated["answers"]["q1"]
+ assert res_gated["answers"]["q2"]["low_confidence"] is True
+
+ # min_confidence=0.80 on predict_batch: q2 flagged, q1 unflagged
+ batch_res = router.predict_batch([request("hello", model="english")], min_confidence=0.80)
+ assert "low_confidence" not in batch_res[0]["answers"]["q1"]
+ assert batch_res[0]["answers"]["q2"]["low_confidence"] is True
+
+ # Rejection of invalid thresholds
+ with pytest.raises(ValueError):
+ router.predict("hello", {"q1": {}}, min_confidence=1.5)
+ with pytest.raises(ValueError):
+ router.predict("hello", {"q1": {}}, min_confidence=True)
+ with pytest.raises(ValueError):
+ router.predict_batch([request("hello", model="english")], min_confidence=-0.1)
+
+
+def test_end_hooks_see_the_low_confidence_flag(monkeypatch):
+ """`on_predict_end` sees `low_confidence` on both Router paths (#361 review)."""
+ import laya.agent
+
+ class Agent:
+ def __init__(self, repo, *, device, token, subfolder):
+ pass
+
+ def system_one(self, state, questions, **overrides):
+ return {
+ "model": "fake",
+ "answers": {
+ "q1": {"type": "choice", "choice": "yes", "answer_confidence": 0.95},
+ "q2": {"type": "choice", "choice": "no", "answer_confidence": 0.60},
+ },
+ }
+
+ def predict_batch(self, states, questions, batch_size=None, **overrides):
+ return [self.system_one(s, questions, **overrides) for s in states]
+
+ monkeypatch.setattr(laya.agent, "Agent", Agent)
+ seen = []
+
+ class Record:
+ def on_predict_end(self, ctx):
+ seen.append({q: a.get("low_confidence", False) for q, a in ctx.results[0]["answers"].items()})
+
+ router = Router(max_loaded=1, default="english", hooks=[Record()])
+
+ router.predict("hello", {"q1": {}}, model="english", min_confidence=0.80)
+ assert seen[-1] == {"q1": False, "q2": True}
+
+ router.predict_batch([request("hello", model="english")], min_confidence=0.80)
+ assert seen[-1] == {"q1": False, "q2": True}
+
+
+def _sort_recording_router(monkeypatch, accepts_sort):
+ """A Router whose attached agent records the sort_by_length it was called with."""
+ import laya.agent
+
+ calls = []
+
+ class Agent:
+ def __init__(self, repo, *, device, token, subfolder):
+ pass
+
+ if accepts_sort:
+ def predict_batch(self, states, questions, batch_size=None, sort_by_length=False,
+ **overrides):
+ calls.append({"n": len(states), "sort": sort_by_length})
+ return [{"model": "fake", "answers": {}, "usage": {}} for _ in states]
+ else:
+ # Strict signature, no **overrides: passing sort_by_length must raise TypeError,
+ # exactly like a real Agent that predates #294.
+ def predict_batch(self, states, questions, batch_size=None):
+ calls.append({"n": len(states), "sort": "unsupported"})
+ return [{"model": "fake", "answers": {}, "usage": {}} for _ in states]
+
+ def system_one(self, state, questions):
+ return self.predict_batch([state], questions)[0]
+
+ monkeypatch.setattr(laya.agent, "Agent", Agent)
+ return Router(max_loaded=1, default="english"), calls
+
+
+def test_sort_by_length_reaches_every_agent_call(monkeypatch):
+ """`Agent.predict_batch` has had `sort_by_length` since #294, but `Router.predict_batch`
+ never forwarded it, so routing -- the normal entry point -- silently lost length grouping.
+ Every model/question-schema group the batch splits into must get the knob."""
+ router, calls = _sort_recording_router(monkeypatch, accepts_sort=True)
+ two_schemas = {"intent": {"type": "noul", "instructions": "Urgent?"}} # different schema text -> own group
+ router.predict_batch([request("a"), request("b"),
+ {"state": "c", "questions": two_schemas}], sort_by_length=True)
+ assert [c["sort"] for c in calls] == [True, True] # one call per group, each with the knob
+ calls.clear()
+ router.predict_batch([request("a"), request("b")]) # default stays off
+ assert [c["sort"] for c in calls] == [False]
+
+
+def test_sort_by_length_is_dropped_for_agents_that_predate_it(monkeypatch):
+ """An attached agent-like object whose `predict_batch` predates #294 must keep serving the
+ batch instead of raising TypeError -- the same tolerance the `lang` forwarding has."""
+ router, calls = _sort_recording_router(monkeypatch, accepts_sort=False)
+ results = router.predict_batch([request("a"), request("b")], sort_by_length=True)
+ assert len(results) == 2
+ assert [c["sort"] for c in calls] == ["unsupported"] # one group, retry without the knob
+ assert [r["routing"]["model"] for r in results] == ["english", "english"]