Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(router): predict_batch dropped each request's lang, so lang_temperatures never applied
PerryLink8dcbc765614e
diff --git a/laya/router.py b/laya/router.pyindex 6becc646..4981fe22 100644--- a/laya/router.py+++ b/laya/router.py@@ -725,11 +725,24 @@ class Router(HookRegistry):overrides = {key: value for key, value in (("max_len", ctx.max_len),("head_max_len", ctx.head_max_len))if value is not None}+ # `predict` forwards the language of the request so the agent can apply its+ # per-language temperatures; the batched path forwarded only the token+ # budgets, so the same request scored differently depending on the entry+ # point. Only computed for an agent that actually carries them: `lang` is+ # otherwise unused, and adding it to the group key would split a group that+ # shares one forward pass today.+ lang_key = None+ if getattr(agent, "lang_temperatures", None):+ lang_key = requests[i].get("lang")+ if lang_key is None:+ detection = decisions[i].get("detection") or {}+ lang_key = detection.get("language")# Order-sensitive at every nesting level (#166): options are positional, so two# equal schemas with different key orders must not share a group.schema = _question_schema(ctx.questions)for group in question_groups:- if group["schema"] == schema and group["overrides"] == overrides:+ if (group["schema"] == schema and group["overrides"] == overrides+ and group["lang"] == lang_key):group["items"].append((i, ctx))breakelse:@@ -737,17 +750,35 @@ class Router(HookRegistry):"questions": ctx.questions,"schema": schema,"overrides": overrides,+ "lang": lang_key,"items": [(i, ctx)],})for group in question_groups:items = group["items"]- batch_results = agent.predict_batch(- [ctx.states[0] for _, ctx in items],- group["questions"],- batch_size=batch_size,- **group["overrides"],- )+ batch_kwargs = dict(group["overrides"])+ if group["lang"] is not None:+ batch_kwargs["lang"] = group["lang"]+ try:+ batch_results = agent.predict_batch(+ [ctx.states[0] for _, ctx in items],+ group["questions"],+ batch_size=batch_size,+ **batch_kwargs,+ )+ except TypeError as e:+ # Same tolerance `predict` has for an Agent-like object whose+ # `predict_batch` predates the `lang` argument.+ if batch_kwargs.get("lang") is not None and "unexpected keyword argument 'lang'" in str(e):+ batch_kwargs.pop("lang")+ batch_results = agent.predict_batch(+ [ctx.states[0] for _, ctx in items],+ group["questions"],+ batch_size=batch_size,+ **batch_kwargs,+ )+ else:+ raiseif len(batch_results) != len(items):raise RuntimeError(diff --git a/tests/test_router_batch.py b/tests/test_router_batch.pyindex 84a2694d..bfdb51a5 100644--- a/tests/test_router_batch.py+++ b/tests/test_router_batch.py@@ -234,3 +234,61 @@ def test_equal_questions_with_different_option_order_score_separately(fake_agent# separate agent calls, each carrying its own caller's option orderorders = [list(call[2]["intent"]["criteria"]) for call in calls]assert orders == [["zulu", "alpha"], ["alpha", "zulu"]]+++def _lang_recording_router(monkeypatch, lang_temperatures):+ """A Router over a fake agent that records the `lang` each batch call received."""+ import laya.agent++ calls = []++ class Agent:+ def __init__(self, repo, *, device, token, subfolder):+ self.checkpoint = subfolder or "english"+ if lang_temperatures is not None:+ self.lang_temperatures = lang_temperatures++ def predict_batch(self, states, questions, batch_size=None, **overrides):+ calls.append({"n": len(states), "lang": overrides.get("lang")})+ return [{"model": "fake", "answers": {}} for _ in states]++ def system_one(self, state, questions, **overrides):+ calls.append({"n": 1, "lang": overrides.get("lang")})+ return {"model": "fake", "answers": {}}++ monkeypatch.setattr(laya.agent, "Agent", Agent)+ return Router(max_loaded=1, default="english"), calls+++def test_lang_reaches_the_agent_when_it_carries_lang_temperatures(monkeypatch):+ """`predict` forwards the request's language so per-language temperatures apply; the batch+ path forwarded only the token budgets, so the same request scored differently depending on+ which entry point served it. The request is routed as German either way, which is what made+ the difference hard to see."""+ router, calls = _lang_recording_router(monkeypatch, {"de": [1.5, 1.5, 1.5]})+ router.predict_batch([request("a", lang="de"), request("b", lang="de"), request("c", lang="fr")])+ assert sorted(c["lang"] for c in calls) == ["de", "fr"]+ # requests sharing a language still share one forward pass+ assert {"n": 2, "lang": "de"} in calls+ assert {"n": 1, "lang": "fr"} in calls+++def test_lang_is_not_added_to_the_group_key_without_lang_temperatures(monkeypatch):+ """An agent with no per-language temperatures does not use `lang`, so naming it must not+ split a group that shares one forward pass today. The explicit `model` keeps every request+ on one checkpoint, so the only thing that could split the group is the lang key."""+ router, calls = _lang_recording_router(monkeypatch, None)+ router.predict_batch([request("a", model="english", lang="de"),+ request("b", model="english", lang="fr"),+ request("c", model="english")])+ assert calls == [{"n": 3, "lang": None}]+++def test_predict_and_predict_batch_pass_the_same_lang(monkeypatch):+ """The invariant that was violated: one request, either entry point, same language."""+ router, calls = _lang_recording_router(monkeypatch, {"de": [1.5, 1.5, 1.5]})+ router.predict("a", Q, model="english", lang="de")+ via_predict = calls[-1]["lang"]+ calls.clear()+ router.predict_batch([request("a", model="english", lang="de")])+ assert calls[-1]["lang"] == via_predict == "de"