Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(router): group heterogeneous requests by checkpoint (#138)

Ahmed Talaat4bcd1f350411
diff --git a/README.md b/README.md
index 05a2d162..593bb96b 100644
--- a/README.md
+++ b/README.md
@@ -246,6 +246,31 @@ router.route({"body": "Esqueci minha senha"}).model # -> multili
router.route({"body": "Please refund the duplicate charge"}).model # -> english
```
+### Heterogeneous routed batches
+
+If a lazy router receives an interleaved workload whose requests route to different checkpoints, calling `predict()` in a loop can still cause unnecessary checkpoint churn when the required checkpoints exceed the resident cache, for example with `max_loaded=1` or when `typed-decisions` is also used.
+
+`Router.predict_batch()` routes the full workload first, groups requests by checkpoint, then groups requests with the same question schema within each checkpoint. Each compatible group is dispatched to `Agent.predict_batch()` so states can share forward passes, and results are restored to the original request order.
+
+```python
+requests = [
+ {"state": "Please refund invoice 1", "questions": questions},
+ {"state": "تم خصم المبلغ مرتين", "questions": questions},
+ {"state": "Please refund invoice 2", "questions": questions},
+]
+
+results = Router(max_loaded=1).predict_batch(requests)
+# results stay in input order while compatible requests are batched by checkpoint
+```
+Each item can independently set `model`, `task`, `lang`, or `lang_guess`. Use `route_batch(requests)` when you only want the ordered routing decisions without loading any checkpoint. `predict_many` is an alias for `predict_batch`.
+
+Requests are validated before model loading. Different requests may use different question schemas; requests sharing both a checkpoint and question schema are passed together to `Agent.predict_batch()`.
+
+You can also bound the Agent-level forward-pass batch size:
+```python
+results = router.predict_batch(requests, batch_size=8)
+```
+
### Why Route: The Evidence
On a shared benchmark (17,416 questions, one T4 GPU, identical questions per model):
diff --git a/laya/router.py b/laya/router.py
index 23cfc3a1..bbdbf6fb 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -31,7 +31,8 @@ import gc
import os
import threading
import time
-from typing import Any, Dict, List, Optional, Union
+from collections.abc import Sequence as SequenceABC
+from typing import Any, Dict, List, Optional, Sequence, Union
from .hooks import HookRegistry, PredictContext, aggregate_usage, dispatch, normalise_hooks
from .lang import analyse
@@ -563,5 +564,133 @@ class Router(HookRegistry):
system_one = predict
+ def route_batch(self, requests: Sequence[Dict[str, Any]]) -> List[RouteDecision]:
+ """Route a heterogeneous request batch without loading any checkpoints.
+
+ Each request is a mapping with ``state`` and ``questions`` plus the same optional
+ routing overrides accepted by :meth:`route`: ``model``, ``task``, ``lang`` and
+ ``lang_guess``. The returned decisions preserve input order.
+
+ This is intentionally separate from inference so callers can inspect or aggregate
+ routing decisions before paying model-load cost.
+ """
+ if not isinstance(requests, SequenceABC) or isinstance(requests, (str, bytes)):
+ raise TypeError("requests must be a sequence of request dictionaries")
+
+ decisions: List[RouteDecision] = []
+ for i, request in enumerate(requests):
+ if not isinstance(request, dict):
+ raise TypeError("request %d must be a dict, got %s" % (i, type(request).__name__))
+ if "state" not in request:
+ raise ValueError("request %d is missing required key 'state'" % i)
+ if "questions" not in request:
+ raise ValueError("request %d is missing required key 'questions'" % i)
+
+ questions = request["questions"]
+ if not isinstance(questions, dict):
+ raise TypeError(
+ "request %d 'questions' must be a dict, got %s"
+ % (i, type(questions).__name__)
+ )
+
+ decisions.append(
+ self.route(
+ request["state"],
+ questions,
+ model=request.get("model"),
+ task=request.get("task"),
+ lang=request.get("lang"),
+ lang_guess=request.get("lang_guess"),
+ )
+ )
+
+ return decisions
+
+ def predict_batch(
+ self,
+ requests: Sequence[Dict[str, Any]],
+ batch_size: Optional[int] = None,
+ ) -> List[Dict[str, Any]]:
+ """Route and execute a heterogeneous request batch with minimal model churn.
+
+ Requests are routed first and grouped by checkpoint. Within each checkpoint,
+ requests that share the same question schema are passed to
+ ``Agent.predict_batch`` so their states can share forward passes. Results are
+ then restored to the original request order.
+
+ Requests may independently specify ``model``, ``task``, ``lang`` or
+ ``lang_guess`` and may use different question schemas.
+
+ Args:
+ requests: Sequence of request dictionaries. Every item requires ``state`` and
+ ``questions`` and may include ``model``, ``task``, ``lang`` or
+ ``lang_guess`` overrides.
+ batch_size: Optional maximum number of states per Agent forward-pass batch.
+
+ Returns:
+ One normal Router prediction result per request, in the same order as the input.
+ """
+ decisions = self.route_batch(requests)
+ if not decisions:
+ return []
+
+ # Dict insertion order preserves the order in which model groups first appear. This
+ # keeps cache effects deterministic while collapsing an arbitrarily interleaved
+ # workload to at most one load per routed checkpoint for this call.
+ groups: Dict[str, List[int]] = {}
+ for i, decision in enumerate(decisions):
+ groups.setdefault(decision.model, []).append(i)
+
+ results: List[Optional[Dict[str, Any]]] = [None] * len(requests)
+
+ for model_name, indices in groups.items():
+ agent = self.load(model_name)
+
+ # Agent.predict_batch evaluates one shared question schema over many states.
+ # Preserve Router's heterogeneous-request API by splitting each checkpoint
+ # group again whenever the question dictionaries differ.
+ question_groups: List[Dict[str, Any]] = []
+ for i in indices:
+ questions = requests[i]["questions"]
+
+ for group in question_groups:
+ if group["questions"] == questions:
+ group["indices"].append(i)
+ break
+ else:
+ question_groups.append({
+ "questions": questions,
+ "indices": [i],
+ })
+
+ for group in question_groups:
+ group_indices = group["indices"]
+ states = [requests[i]["state"] for i in group_indices]
+
+ batch_results = agent.predict_batch(
+ states,
+ group["questions"],
+ batch_size=batch_size,
+ )
+
+ if len(batch_results) != len(group_indices):
+ raise RuntimeError(
+ "internal error: Agent.predict_batch returned %d results for %d states"
+ % (len(batch_results), len(group_indices))
+ )
+
+ for i, result in zip(group_indices, batch_results):
+ result["routing"] = dict(decisions[i])
+ results[i] = result
+
+ # Every input index is assigned exactly once by construction. Keep this assertion local
+ # so a future refactor cannot silently return a partially-filled batch.
+ if any(result is None for result in results):
+ raise RuntimeError("internal error: batch execution did not produce every result")
+
+ return [result for result in results if result is not None]
+
+ predict_many = predict_batch
+
def __repr__(self):
return "Router(loaded=%s, max_loaded=%d, default=%r)" % (self.loaded, self.max_loaded, self.default)
diff --git a/research/scripts/bench_router_grouping.py b/research/scripts/bench_router_grouping.py
new file mode 100644
index 00000000..909e29a3
--- /dev/null
+++ b/research/scripts/bench_router_grouping.py
@@ -0,0 +1,85 @@
+"""Synthetic Router orchestration benchmark; no weights or model inference.
+
+Run with ``python research/scripts/bench_router_grouping.py``. Cold loads have a fixed
+ delay to illustrate cache churn; elapsed times depend on the host and are not a model
+throughput or latency measurement.
+"""
+import argparse
+import os
+import sys
+import time
+from unittest.mock import patch
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from laya.router import Router # noqa: E402
+
+
+Q = {"intent": {"type": "noul", "instructions": "Relevant?"}}
+
+
+def requests(count):
+ return [{"state": "request %d" % i, "questions": Q,
+ "model": "english" if i % 2 == 0 else "multilingual"}
+ for i in range(count)]
+
+
+def measure(items, capacity, delay, grouped):
+ constructions = []
+ evictions = []
+
+ class StubAgent:
+ def __init__(self, repo, *, device, token, subfolder):
+ key = subfolder or "english"
+ constructions.append(key)
+ time.sleep(delay)
+
+ def system_one(self, state, questions):
+ return {"model": "stub", "answers": {"seen": state}, "usage": {}}
+
+ with patch("laya.agent.Agent", StubAgent):
+ router = Router(max_loaded=capacity)
+ original_evict = router._evict
+
+ def count_evictions():
+ before = set(router.loaded)
+ original_evict()
+ evictions.extend(before - set(router.loaded))
+
+ router._evict = count_evictions
+ start = time.perf_counter()
+ if grouped:
+ outputs = router.predict_batch(items)
+ else:
+ outputs = [router.predict(item["state"], item["questions"], model=item["model"])
+ for item in items]
+ elapsed = time.perf_counter() - start
+
+ expected = [item["state"] for item in items]
+ correct = [output["answers"]["seen"] for output in outputs] == expected
+ correct &= [output["routing"]["model"] for output in outputs] == [item["model"] for item in items]
+ return len(constructions), len(evictions), elapsed, correct
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--requests", type=int, default=100)
+ parser.add_argument("--load-delay", type=float, default=0.01,
+ help="seconds per synthetic cold load")
+ args = parser.parse_args()
+ if args.requests < 0 or args.load_delay < 0:
+ parser.error("requests and load-delay must be non-negative")
+ items = requests(args.requests)
+ print("synthetic orchestration only; requests=%d; cold-load delay=%.3f s" %
+ (len(items), args.load_delay))
+ print("max_loaded method constructions evictions elapsed_s ordered")
+ for capacity in (1, 2):
+ for name, grouped in (("repeated_predict", False), ("predict_batch", True)):
+ loads, evictions, elapsed, correct = measure(items, capacity, args.load_delay, grouped)
+ print("%d %s %d %d %.6f %s" % (capacity, name, loads, evictions, elapsed, correct))
+ if not correct:
+ raise AssertionError("result ordering or routing differs from input")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/test_router_batch.py b/tests/test_router_batch.py
new file mode 100644
index 00000000..48c45463
--- /dev/null
+++ b/tests/test_router_batch.py
@@ -0,0 +1,217 @@
+"""Deterministic regression coverage for heterogeneous Router batching."""
+
+from concurrent.futures import ThreadPoolExecutor
+from threading import Barrier, Lock
+from time import sleep
+
+import pytest
+
+from laya.router import Router
+
+
+Q = {"intent": {"type": "noul", "instructions": "Relevant?"}}
+
+
+def request(state, **overrides):
+ return {"state": state, "questions": Q, **overrides}
+
+
+def fake_agent(monkeypatch):
+ import laya.agent
+
+ built = []
+ calls = []
+
+ class Agent:
+ def __init__(self, repo, *, device, token, subfolder):
+ self.checkpoint = subfolder or "english"
+ built.append(self.checkpoint)
+
+ def predict_batch(self, states, questions, batch_size=None):
+ calls.append((self.checkpoint, list(states), questions, batch_size))
+ if "raise" in states:
+ raise RuntimeError("inference failed")
+ return [
+ {"model": "laya-rl-agent", "answers": {"seen": state}, "usage": {}}
+ for state in states
+ ]
+
+ def system_one(self, state, questions):
+ return self.predict_batch([state], questions)[0]
+
+ monkeypatch.setattr(laya.agent, "Agent", Agent)
+ return built, calls
+
+
[email protected]("capacity,expected", [
+ (1, (["english", "multilingual", "typed-decisions"], ["typed-decisions"])),
+ (2, (["english", "multilingual", "typed-decisions"], ["multilingual", "typed-decisions"])),
+ (3, (["english", "multilingual", "typed-decisions"],
+ ["english", "multilingual", "typed-decisions"])),
+])
+def test_mixed_groups_keep_order_and_lru(fake_agent, capacity, expected):
+ built, calls = fake_agent
+ router = Router(max_loaded=capacity, auto_task_detection=True)
+ typed = {key: {"type": "noul", "instructions": "?"} for key in
+ ("action", "needs_review", "outcome", "risk", "urgency")}
+ items = [request("English text one"), request("مرحبا"),
+ request("English text two"), {"state": "decision", "questions": typed},
+ request("forced", model="ml"), request("forced english", lang="en"),
+ request("explicit task", task="typed_decisions")]
+ decisions = router.route_batch(items)
+ assert router.loaded == []
+ results = router.predict_batch(items)
+ assert [r["answers"]["seen"] for r in results] == [r["state"] for r in items]
+ assert [r["routing"] for r in results] == list(map(dict, decisions))
+ assert built == expected[0]
+ assert router.loaded == expected[1]
+ assert [(c[0], c[1]) for c in calls] == [
+ ("english", ["English text one", "English text two", "forced english"]),
+ ("multilingual", ["مرحبا", "forced"]),
+ ("typed-decisions", ["decision"]),
+ ("typed-decisions", ["explicit task"]),
+ ]
+ assert router.predict_many([]) == []
+ assert router.route_batch(()) == []
+
+
[email protected]("items,error,fragment", [
+ (None, TypeError, "requests must be a sequence"),
+ ({}, TypeError, "requests must be a sequence"),
+ ("text", TypeError, "requests must be a sequence"),
+ ([None], TypeError, "request 0"),
+ ([{"questions": Q}], ValueError, "request 0 is missing required key 'state'"),
+ ([{"state": "x"}], ValueError, "request 0 is missing required key 'questions'"),
+ ([request("x"), {"state": "y", "questions": None}], TypeError, "request 1 'questions'"),
+ ([request("x"), request("y", model="invalid")], ValueError, "unknown model"),
+])
+def test_invalid_batch_fails_before_loading(fake_agent, items, error, fragment):
+ built, _ = fake_agent
+ router = Router()
+ with pytest.raises(error, match=fragment):
+ router.predict_batch(items)
+ assert built == []
+ assert router.loaded == []
+
+
+def test_inference_exception_propagates_and_cache_remains_consistent(fake_agent):
+ built, calls = fake_agent
+ router = Router(max_loaded=1)
+ with pytest.raises(RuntimeError, match="inference failed"):
+ router.predict_batch([request("first"), request("raise", lang="ar"),
+ request("unreached", lang="ar")])
+ assert built == ["english", "multilingual"]
+ assert [(c[0], c[1]) for c in calls] == [
+ ("english", ["first"]),
+ ("multilingual", ["raise", "unreached"]),
+ ]
+ assert router.loaded == ["multilingual"]
+ assert list(router._agents) == router.loaded
+ assert router.predict("after failure", Q, lang="ar")["answers"]["seen"] == "after failure"
+
+
+def test_warm_cache_and_repeated_batches(fake_agent):
+ built, _ = fake_agent
+ router = Router(max_loaded=2)
+ router.load("multilingual")
+ result = router.predict_batch([request("en", model="english"),
+ request("ar", model="multilingual"),
+ request("en again", model="english")])
+ assert [r["answers"]["seen"] for r in result] == ["en", "ar", "en again"]
+ assert built == ["multilingual", "english"]
+ assert router.loaded == ["english", "multilingual"]
+ router.predict_batch([request("ar", model="multilingual"), request("en", model="english")])
+ assert built == ["multilingual", "english"]
+
+
+
+def test_same_checkpoint_same_questions_uses_one_agent_batch(fake_agent):
+ _, calls = fake_agent
+ router = Router(max_loaded=2)
+ items = [
+ request("one", model="english"),
+ request("two", model="english"),
+ request("three", model="english"),
+ ]
+
+ results = router.predict_batch(items, batch_size=2)
+
+ assert [r["answers"]["seen"] for r in results] == ["one", "two", "three"]
+ assert len(calls) == 1
+ assert calls[0][0] == "english"
+ assert calls[0][1] == ["one", "two", "three"]
+ assert calls[0][2] == Q
+ assert calls[0][3] == 2
+
+
+def test_same_checkpoint_different_questions_split_agent_batches(fake_agent):
+ _, calls = fake_agent
+ router = Router(max_loaded=2)
+ q2 = {"risk": {"type": "noul", "instructions": "Risky?"}}
+ items = [
+ {"state": "one", "questions": Q, "model": "english"},
+ {"state": "two", "questions": q2, "model": "english"},
+ {"state": "three", "questions": Q, "model": "english"},
+ ]
+
+ results = router.predict_batch(items)
+
+ assert [r["answers"]["seen"] for r in results] == ["one", "two", "three"]
+ assert len(calls) == 2
+ assert calls[0][0] == "english"
+ assert calls[0][1] == ["one", "three"]
+ assert calls[0][2] == Q
+ assert calls[1][0] == "english"
+ assert calls[1][1] == ["two"]
+ assert calls[1][2] == q2
+
+
+def test_route_batch_forwards_lang_guess(fake_agent):
+ built, _ = fake_agent
+ router = Router()
+
+ decisions = router.route_batch([
+ request("hola", lang_guess="es"),
+ request("hello", lang_guess="en-US"),
+ ])
+
+ assert [decision.model for decision in decisions] == ["multilingual", "english"]
+ assert built == []
+
+def test_concurrent_batch_load_deduplicates(monkeypatch):
+ import laya.agent
+
+ built = []
+ guard = Lock()
+
+ class SlowAgent:
+ def __init__(self, repo, *, device, token, subfolder):
+ sleep(0.01)
+ with guard:
+ built.append(subfolder or "english")
+
+ def predict_batch(self, states, questions, batch_size=None):
+ return [
+ {"model": "stub", "answers": {"seen": state}, "usage": {}}
+ for state in states
+ ]
+
+ def system_one(self, state, questions):
+ return self.predict_batch([state], questions)[0]
+
+ monkeypatch.setattr(laya.agent, "Agent", SlowAgent)
+ router = Router(max_loaded=2)
+ start = Barrier(8)
+
+ def worker(_):
+ start.wait()
+ return router.predict_batch([request("hello", model="english"),
+ request("مرحبا", model="multilingual")])
+
+ with ThreadPoolExecutor(max_workers=8) as pool:
+ results = list(pool.map(worker, range(8)))
+ assert built == ["english", "multilingual"]
+ assert router.loaded == ["english", "multilingual"]
+ assert all([r["answers"]["seen"] for r in result] == ["hello", "مرحبا"]
+ for result in results)