Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #509 from aashish254/feat-router-sort

Nandakishore8d63366a85b
diff --cc laya/router.py
index d7bff65e,3d1b8b10..22c30cc6
--- a/laya/router.py
+++ b/laya/router.py
@@@ -790,7 -704,7 +790,8 @@@ class Router(HookRegistry)
requests: Sequence[Dict[str, Any]],
batch_size: Optional[int] = None,
hooks_timeout: Optional[float] = None,
+ min_confidence: Optional[float] = None,
+ sort_by_length: bool = False,
) -> List[Dict[str, Any]]:
"""Route and execute a heterogeneous request batch with minimal model churn.
diff --cc tests/test_router_batch.py
index 21628bce,33815034..1fbd008a
--- a/tests/test_router_batch.py
+++ b/tests/test_router_batch.py
@@@ -308,84 -308,54 +308,137 @@@ def test_predict_and_predict_batch_pass
assert calls[-1]["lang"] == via_predict == "de"
+def test_router_predict_and_predict_batch_min_confidence(monkeypatch):
+ """`min_confidence` gates answers below threshold on both predict and predict_batch (#361)."""
+ import laya.agent
+
+ class Agent:
+ def __init__(self, repo, *, device, token, subfolder):
+ pass
+
+ def system_one(self, state, questions, **overrides):
+ return {
+ "model": "fake",
+ "answers": {
+ "q1": {"type": "choice", "choice": "yes", "answer_confidence": 0.95},
+ "q2": {"type": "choice", "choice": "no", "answer_confidence": 0.60},
+ },
+ }
+
+ def predict_batch(self, states, questions, batch_size=None, **overrides):
+ return [self.system_one(s, questions, **overrides) for s in states]
+
+ monkeypatch.setattr(laya.agent, "Agent", Agent)
+ router = Router(max_loaded=1, default="english")
+
+ # Default min_confidence=None: no low_confidence flags
+ res_default = router.predict("hello", {"q1": {}}, model="english")
+ assert "low_confidence" not in res_default["answers"]["q1"]
+ assert "low_confidence" not in res_default["answers"]["q2"]
+
+ # min_confidence=0.80 on predict: q2 flagged, q1 unflagged
+ res_gated = router.predict("hello", {"q1": {}}, model="english", min_confidence=0.80)
+ assert "low_confidence" not in res_gated["answers"]["q1"]
+ assert res_gated["answers"]["q2"]["low_confidence"] is True
+
+ # min_confidence=0.80 on predict_batch: q2 flagged, q1 unflagged
+ batch_res = router.predict_batch([request("hello", model="english")], min_confidence=0.80)
+ assert "low_confidence" not in batch_res[0]["answers"]["q1"]
+ assert batch_res[0]["answers"]["q2"]["low_confidence"] is True
+
+ # Rejection of invalid thresholds
+ with pytest.raises(ValueError):
+ router.predict("hello", {"q1": {}}, min_confidence=1.5)
+ with pytest.raises(ValueError):
+ router.predict("hello", {"q1": {}}, min_confidence=True)
+ with pytest.raises(ValueError):
+ router.predict_batch([request("hello", model="english")], min_confidence=-0.1)
+
+
+def test_end_hooks_see_the_low_confidence_flag(monkeypatch):
+ """`on_predict_end` sees `low_confidence` on both Router paths (#361 review)."""
+ import laya.agent
+
+ class Agent:
+ def __init__(self, repo, *, device, token, subfolder):
+ pass
+
+ def system_one(self, state, questions, **overrides):
+ return {
+ "model": "fake",
+ "answers": {
+ "q1": {"type": "choice", "choice": "yes", "answer_confidence": 0.95},
+ "q2": {"type": "choice", "choice": "no", "answer_confidence": 0.60},
+ },
+ }
+
+ def predict_batch(self, states, questions, batch_size=None, **overrides):
+ return [self.system_one(s, questions, **overrides) for s in states]
+
+ monkeypatch.setattr(laya.agent, "Agent", Agent)
+ seen = []
+
+ class Record:
+ def on_predict_end(self, ctx):
+ seen.append({q: a.get("low_confidence", False) for q, a in ctx.results[0]["answers"].items()})
+
+ router = Router(max_loaded=1, default="english", hooks=[Record()])
+
+ router.predict("hello", {"q1": {}}, model="english", min_confidence=0.80)
+ assert seen[-1] == {"q1": False, "q2": True}
+
+ router.predict_batch([request("hello", model="english")], min_confidence=0.80)
+ assert seen[-1] == {"q1": False, "q2": True}
++
++
+ def _sort_recording_router(monkeypatch, accepts_sort):
+ """A Router whose attached agent records the sort_by_length it was called with."""
+ import laya.agent
+
+ calls = []
+
+ class Agent:
+ def __init__(self, repo, *, device, token, subfolder):
+ pass
+
+ if accepts_sort:
+ def predict_batch(self, states, questions, batch_size=None, sort_by_length=False,
+ **overrides):
+ calls.append({"n": len(states), "sort": sort_by_length})
+ return [{"model": "fake", "answers": {}, "usage": {}} for _ in states]
+ else:
+ # Strict signature, no **overrides: passing sort_by_length must raise TypeError,
+ # exactly like a real Agent that predates #294.
+ def predict_batch(self, states, questions, batch_size=None):
+ calls.append({"n": len(states), "sort": "unsupported"})
+ return [{"model": "fake", "answers": {}, "usage": {}} for _ in states]
+
+ def system_one(self, state, questions):
+ return self.predict_batch([state], questions)[0]
+
+ monkeypatch.setattr(laya.agent, "Agent", Agent)
+ return Router(max_loaded=1, default="english"), calls
+
+
+ def test_sort_by_length_reaches_every_agent_call(monkeypatch):
+ """`Agent.predict_batch` has had `sort_by_length` since #294, but `Router.predict_batch`
+ never forwarded it, so routing -- the normal entry point -- silently lost length grouping.
+ Every model/question-schema group the batch splits into must get the knob."""
+ router, calls = _sort_recording_router(monkeypatch, accepts_sort=True)
+ two_schemas = {"intent": {"type": "noul", "instructions": "Urgent?"}} # different schema text -> own group
+ router.predict_batch([request("a"), request("b"),
+ {"state": "c", "questions": two_schemas}], sort_by_length=True)
+ assert [c["sort"] for c in calls] == [True, True] # one call per group, each with the knob
+ calls.clear()
+ router.predict_batch([request("a"), request("b")]) # default stays off
+ assert [c["sort"] for c in calls] == [False]
+
+
+ def test_sort_by_length_is_dropped_for_agents_that_predate_it(monkeypatch):
+ """An attached agent-like object whose `predict_batch` predates #294 must keep serving the
+ batch instead of raising TypeError -- the same tolerance the `lang` forwarding has."""
+ router, calls = _sort_recording_router(monkeypatch, accepts_sort=False)
+ results = router.predict_batch([request("a"), request("b")], sort_by_length=True)
+ assert len(results) == 2
+ assert [c["sort"] for c in calls] == ["unsupported"] # one group, retry without the knob
+ assert [r["routing"]["model"] for r in results] == ["english", "english"]