Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(router): group heterogeneous requests by checkpoint (#138)
Ahmed Talaat4bcd1f350411
diff --git a/README.md b/README.mdindex 05a2d162..593bb96b 100644--- a/README.md+++ b/README.md@@ -246,6 +246,31 @@ router.route({"body": "Esqueci minha senha"}).model # -> multilirouter.route({"body": "Please refund the duplicate charge"}).model # -> english```+### Heterogeneous routed batches++If a lazy router receives an interleaved workload whose requests route to different checkpoints, calling `predict()` in a loop can still cause unnecessary checkpoint churn when the required checkpoints exceed the resident cache, for example with `max_loaded=1` or when `typed-decisions` is also used.++`Router.predict_batch()` routes the full workload first, groups requests by checkpoint, then groups requests with the same question schema within each checkpoint. Each compatible group is dispatched to `Agent.predict_batch()` so states can share forward passes, and results are restored to the original request order.++```python+requests = [+ {"state": "Please refund invoice 1", "questions": questions},+ {"state": "تم خصم المبلغ مرتين", "questions": questions},+ {"state": "Please refund invoice 2", "questions": questions},+]++results = Router(max_loaded=1).predict_batch(requests)+# results stay in input order while compatible requests are batched by checkpoint+```+Each item can independently set `model`, `task`, `lang`, or `lang_guess`. Use `route_batch(requests)` when you only want the ordered routing decisions without loading any checkpoint. `predict_many` is an alias for `predict_batch`.++Requests are validated before model loading. Different requests may use different question schemas; requests sharing both a checkpoint and question schema are passed together to `Agent.predict_batch()`.++You can also bound the Agent-level forward-pass batch size:+```python+results = router.predict_batch(requests, batch_size=8)+```+### Why Route: The EvidenceOn a shared benchmark (17,416 questions, one T4 GPU, identical questions per model):diff --git a/laya/router.py b/laya/router.pyindex 23cfc3a1..bbdbf6fb 100644--- a/laya/router.py+++ b/laya/router.py@@ -31,7 +31,8 @@ import gcimport osimport threadingimport time-from typing import Any, Dict, List, Optional, Union+from collections.abc import Sequence as SequenceABC+from typing import Any, Dict, List, Optional, Sequence, Unionfrom .hooks import HookRegistry, PredictContext, aggregate_usage, dispatch, normalise_hooksfrom .lang import analyse@@ -563,5 +564,133 @@ class Router(HookRegistry):system_one = predict+ def route_batch(self, requests: Sequence[Dict[str, Any]]) -> List[RouteDecision]:+ """Route a heterogeneous request batch without loading any checkpoints.++ Each request is a mapping with ``state`` and ``questions`` plus the same optional+ routing overrides accepted by :meth:`route`: ``model``, ``task``, ``lang`` and+ ``lang_guess``. The returned decisions preserve input order.++ This is intentionally separate from inference so callers can inspect or aggregate+ routing decisions before paying model-load cost.+ """+ if not isinstance(requests, SequenceABC) or isinstance(requests, (str, bytes)):+ raise TypeError("requests must be a sequence of request dictionaries")++ decisions: List[RouteDecision] = []+ for i, request in enumerate(requests):+ if not isinstance(request, dict):+ raise TypeError("request %d must be a dict, got %s" % (i, type(request).__name__))+ if "state" not in request:+ raise ValueError("request %d is missing required key 'state'" % i)+ if "questions" not in request:+ raise ValueError("request %d is missing required key 'questions'" % i)++ questions = request["questions"]+ if not isinstance(questions, dict):+ raise TypeError(+ "request %d 'questions' must be a dict, got %s"+ % (i, type(questions).__name__)+ )++ decisions.append(+ self.route(+ request["state"],+ questions,+ model=request.get("model"),+ task=request.get("task"),+ lang=request.get("lang"),+ lang_guess=request.get("lang_guess"),+ )+ )++ return decisions++ def predict_batch(+ self,+ requests: Sequence[Dict[str, Any]],+ batch_size: Optional[int] = None,+ ) -> List[Dict[str, Any]]:+ """Route and execute a heterogeneous request batch with minimal model churn.++ Requests are routed first and grouped by checkpoint. Within each checkpoint,+ requests that share the same question schema are passed to+ ``Agent.predict_batch`` so their states can share forward passes. Results are+ then restored to the original request order.++ Requests may independently specify ``model``, ``task``, ``lang`` or+ ``lang_guess`` and may use different question schemas.++ Args:+ requests: Sequence of request dictionaries. Every item requires ``state`` and+ ``questions`` and may include ``model``, ``task``, ``lang`` or+ ``lang_guess`` overrides.+ batch_size: Optional maximum number of states per Agent forward-pass batch.++ Returns:+ One normal Router prediction result per request, in the same order as the input.+ """+ decisions = self.route_batch(requests)+ if not decisions:+ return []++ # Dict insertion order preserves the order in which model groups first appear. This+ # keeps cache effects deterministic while collapsing an arbitrarily interleaved+ # workload to at most one load per routed checkpoint for this call.+ groups: Dict[str, List[int]] = {}+ for i, decision in enumerate(decisions):+ groups.setdefault(decision.model, []).append(i)++ results: List[Optional[Dict[str, Any]]] = [None] * len(requests)++ for model_name, indices in groups.items():+ agent = self.load(model_name)++ # Agent.predict_batch evaluates one shared question schema over many states.+ # Preserve Router's heterogeneous-request API by splitting each checkpoint+ # group again whenever the question dictionaries differ.+ question_groups: List[Dict[str, Any]] = []+ for i in indices:+ questions = requests[i]["questions"]++ for group in question_groups:+ if group["questions"] == questions:+ group["indices"].append(i)+ break+ else:+ question_groups.append({+ "questions": questions,+ "indices": [i],+ })++ for group in question_groups:+ group_indices = group["indices"]+ states = [requests[i]["state"] for i in group_indices]++ batch_results = agent.predict_batch(+ states,+ group["questions"],+ batch_size=batch_size,+ )++ if len(batch_results) != len(group_indices):+ raise RuntimeError(+ "internal error: Agent.predict_batch returned %d results for %d states"+ % (len(batch_results), len(group_indices))+ )++ for i, result in zip(group_indices, batch_results):+ result["routing"] = dict(decisions[i])+ results[i] = result++ # Every input index is assigned exactly once by construction. Keep this assertion local+ # so a future refactor cannot silently return a partially-filled batch.+ if any(result is None for result in results):+ raise RuntimeError("internal error: batch execution did not produce every result")++ return [result for result in results if result is not None]++ predict_many = predict_batch+def __repr__(self):return "Router(loaded=%s, max_loaded=%d, default=%r)" % (self.loaded, self.max_loaded, self.default)diff --git a/research/scripts/bench_router_grouping.py b/research/scripts/bench_router_grouping.pynew file mode 100644index 00000000..909e29a3--- /dev/null+++ b/research/scripts/bench_router_grouping.py@@ -0,0 +1,85 @@+"""Synthetic Router orchestration benchmark; no weights or model inference.++Run with ``python research/scripts/bench_router_grouping.py``. Cold loads have a fixed+ delay to illustrate cache churn; elapsed times depend on the host and are not a model+throughput or latency measurement.+"""+import argparse+import os+import sys+import time+from unittest.mock import patch++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))++from laya.router import Router # noqa: E402+++Q = {"intent": {"type": "noul", "instructions": "Relevant?"}}+++def requests(count):+ return [{"state": "request %d" % i, "questions": Q,+ "model": "english" if i % 2 == 0 else "multilingual"}+ for i in range(count)]+++def measure(items, capacity, delay, grouped):+ constructions = []+ evictions = []++ class StubAgent:+ def __init__(self, repo, *, device, token, subfolder):+ key = subfolder or "english"+ constructions.append(key)+ time.sleep(delay)++ def system_one(self, state, questions):+ return {"model": "stub", "answers": {"seen": state}, "usage": {}}++ with patch("laya.agent.Agent", StubAgent):+ router = Router(max_loaded=capacity)+ original_evict = router._evict++ def count_evictions():+ before = set(router.loaded)+ original_evict()+ evictions.extend(before - set(router.loaded))++ router._evict = count_evictions+ start = time.perf_counter()+ if grouped:+ outputs = router.predict_batch(items)+ else:+ outputs = [router.predict(item["state"], item["questions"], model=item["model"])+ for item in items]+ elapsed = time.perf_counter() - start++ expected = [item["state"] for item in items]+ correct = [output["answers"]["seen"] for output in outputs] == expected+ correct &= [output["routing"]["model"] for output in outputs] == [item["model"] for item in items]+ return len(constructions), len(evictions), elapsed, correct+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--requests", type=int, default=100)+ parser.add_argument("--load-delay", type=float, default=0.01,+ help="seconds per synthetic cold load")+ args = parser.parse_args()+ if args.requests < 0 or args.load_delay < 0:+ parser.error("requests and load-delay must be non-negative")+ items = requests(args.requests)+ print("synthetic orchestration only; requests=%d; cold-load delay=%.3f s" %+ (len(items), args.load_delay))+ print("max_loaded method constructions evictions elapsed_s ordered")+ for capacity in (1, 2):+ for name, grouped in (("repeated_predict", False), ("predict_batch", True)):+ loads, evictions, elapsed, correct = measure(items, capacity, args.load_delay, grouped)+ print("%d %s %d %d %.6f %s" % (capacity, name, loads, evictions, elapsed, correct))+ if not correct:+ raise AssertionError("result ordering or routing differs from input")+++if __name__ == "__main__":+ main()diff --git a/tests/test_router_batch.py b/tests/test_router_batch.pynew file mode 100644index 00000000..48c45463--- /dev/null+++ b/tests/test_router_batch.py@@ -0,0 +1,217 @@+"""Deterministic regression coverage for heterogeneous Router batching."""++from concurrent.futures import ThreadPoolExecutor+from threading import Barrier, Lock+from time import sleep++import pytest++from laya.router import Router+++Q = {"intent": {"type": "noul", "instructions": "Relevant?"}}+++def request(state, **overrides):+ return {"state": state, "questions": Q, **overrides}+++def fake_agent(monkeypatch):+ import laya.agent++ built = []+ calls = []++ class Agent:+ def __init__(self, repo, *, device, token, subfolder):+ self.checkpoint = subfolder or "english"+ built.append(self.checkpoint)++ def predict_batch(self, states, questions, batch_size=None):+ calls.append((self.checkpoint, list(states), questions, batch_size))+ if "raise" in states:+ raise RuntimeError("inference failed")+ return [+ {"model": "laya-rl-agent", "answers": {"seen": state}, "usage": {}}+ for state in states+ ]++ def system_one(self, state, questions):+ return self.predict_batch([state], questions)[0]++ monkeypatch.setattr(laya.agent, "Agent", Agent)+ return built, calls++[email protected]("capacity,expected", [+ (1, (["english", "multilingual", "typed-decisions"], ["typed-decisions"])),+ (2, (["english", "multilingual", "typed-decisions"], ["multilingual", "typed-decisions"])),+ (3, (["english", "multilingual", "typed-decisions"],+ ["english", "multilingual", "typed-decisions"])),+])+def test_mixed_groups_keep_order_and_lru(fake_agent, capacity, expected):+ built, calls = fake_agent+ router = Router(max_loaded=capacity, auto_task_detection=True)+ typed = {key: {"type": "noul", "instructions": "?"} for key in+ ("action", "needs_review", "outcome", "risk", "urgency")}+ items = [request("English text one"), request("مرحبا"),+ request("English text two"), {"state": "decision", "questions": typed},+ request("forced", model="ml"), request("forced english", lang="en"),+ request("explicit task", task="typed_decisions")]+ decisions = router.route_batch(items)+ assert router.loaded == []+ results = router.predict_batch(items)+ assert [r["answers"]["seen"] for r in results] == [r["state"] for r in items]+ assert [r["routing"] for r in results] == list(map(dict, decisions))+ assert built == expected[0]+ assert router.loaded == expected[1]+ assert [(c[0], c[1]) for c in calls] == [+ ("english", ["English text one", "English text two", "forced english"]),+ ("multilingual", ["مرحبا", "forced"]),+ ("typed-decisions", ["decision"]),+ ("typed-decisions", ["explicit task"]),+ ]+ assert router.predict_many([]) == []+ assert router.route_batch(()) == []++[email protected]("items,error,fragment", [+ (None, TypeError, "requests must be a sequence"),+ ({}, TypeError, "requests must be a sequence"),+ ("text", TypeError, "requests must be a sequence"),+ ([None], TypeError, "request 0"),+ ([{"questions": Q}], ValueError, "request 0 is missing required key 'state'"),+ ([{"state": "x"}], ValueError, "request 0 is missing required key 'questions'"),+ ([request("x"), {"state": "y", "questions": None}], TypeError, "request 1 'questions'"),+ ([request("x"), request("y", model="invalid")], ValueError, "unknown model"),+])+def test_invalid_batch_fails_before_loading(fake_agent, items, error, fragment):+ built, _ = fake_agent+ router = Router()+ with pytest.raises(error, match=fragment):+ router.predict_batch(items)+ assert built == []+ assert router.loaded == []+++def test_inference_exception_propagates_and_cache_remains_consistent(fake_agent):+ built, calls = fake_agent+ router = Router(max_loaded=1)+ with pytest.raises(RuntimeError, match="inference failed"):+ router.predict_batch([request("first"), request("raise", lang="ar"),+ request("unreached", lang="ar")])+ assert built == ["english", "multilingual"]+ assert [(c[0], c[1]) for c in calls] == [+ ("english", ["first"]),+ ("multilingual", ["raise", "unreached"]),+ ]+ assert router.loaded == ["multilingual"]+ assert list(router._agents) == router.loaded+ assert router.predict("after failure", Q, lang="ar")["answers"]["seen"] == "after failure"+++def test_warm_cache_and_repeated_batches(fake_agent):+ built, _ = fake_agent+ router = Router(max_loaded=2)+ router.load("multilingual")+ result = router.predict_batch([request("en", model="english"),+ request("ar", model="multilingual"),+ request("en again", model="english")])+ assert [r["answers"]["seen"] for r in result] == ["en", "ar", "en again"]+ assert built == ["multilingual", "english"]+ assert router.loaded == ["english", "multilingual"]+ router.predict_batch([request("ar", model="multilingual"), request("en", model="english")])+ assert built == ["multilingual", "english"]++++def test_same_checkpoint_same_questions_uses_one_agent_batch(fake_agent):+ _, calls = fake_agent+ router = Router(max_loaded=2)+ items = [+ request("one", model="english"),+ request("two", model="english"),+ request("three", model="english"),+ ]++ results = router.predict_batch(items, batch_size=2)++ assert [r["answers"]["seen"] for r in results] == ["one", "two", "three"]+ assert len(calls) == 1+ assert calls[0][0] == "english"+ assert calls[0][1] == ["one", "two", "three"]+ assert calls[0][2] == Q+ assert calls[0][3] == 2+++def test_same_checkpoint_different_questions_split_agent_batches(fake_agent):+ _, calls = fake_agent+ router = Router(max_loaded=2)+ q2 = {"risk": {"type": "noul", "instructions": "Risky?"}}+ items = [+ {"state": "one", "questions": Q, "model": "english"},+ {"state": "two", "questions": q2, "model": "english"},+ {"state": "three", "questions": Q, "model": "english"},+ ]++ results = router.predict_batch(items)++ assert [r["answers"]["seen"] for r in results] == ["one", "two", "three"]+ assert len(calls) == 2+ assert calls[0][0] == "english"+ assert calls[0][1] == ["one", "three"]+ assert calls[0][2] == Q+ assert calls[1][0] == "english"+ assert calls[1][1] == ["two"]+ assert calls[1][2] == q2+++def test_route_batch_forwards_lang_guess(fake_agent):+ built, _ = fake_agent+ router = Router()++ decisions = router.route_batch([+ request("hola", lang_guess="es"),+ request("hello", lang_guess="en-US"),+ ])++ assert [decision.model for decision in decisions] == ["multilingual", "english"]+ assert built == []++def test_concurrent_batch_load_deduplicates(monkeypatch):+ import laya.agent++ built = []+ guard = Lock()++ class SlowAgent:+ def __init__(self, repo, *, device, token, subfolder):+ sleep(0.01)+ with guard:+ built.append(subfolder or "english")++ def predict_batch(self, states, questions, batch_size=None):+ return [+ {"model": "stub", "answers": {"seen": state}, "usage": {}}+ for state in states+ ]++ def system_one(self, state, questions):+ return self.predict_batch([state], questions)[0]++ monkeypatch.setattr(laya.agent, "Agent", SlowAgent)+ router = Router(max_loaded=2)+ start = Barrier(8)++ def worker(_):+ start.wait()+ return router.predict_batch([request("hello", model="english"),+ request("مرحبا", model="multilingual")])++ with ThreadPoolExecutor(max_workers=8) as pool:+ results = list(pool.map(worker, range(8)))+ assert built == ["english", "multilingual"]+ assert router.loaded == ["english", "multilingual"]+ assert all([r["answers"]["seen"] for r in result] == ["hello", "مرحبا"]+ for result in results)