Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(router): predict_batch dropped each request's lang, so lang_temperatures never applied

PerryLink8dcbc765614e
diff --git a/laya/router.py b/laya/router.py
index 6becc646..4981fe22 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -725,11 +725,24 @@ class Router(HookRegistry):
overrides = {key: value for key, value in (("max_len", ctx.max_len),
("head_max_len", ctx.head_max_len))
if value is not None}
+ # `predict` forwards the language of the request so the agent can apply its
+ # per-language temperatures; the batched path forwarded only the token
+ # budgets, so the same request scored differently depending on the entry
+ # point. Only computed for an agent that actually carries them: `lang` is
+ # otherwise unused, and adding it to the group key would split a group that
+ # shares one forward pass today.
+ lang_key = None
+ if getattr(agent, "lang_temperatures", None):
+ lang_key = requests[i].get("lang")
+ if lang_key is None:
+ detection = decisions[i].get("detection") or {}
+ lang_key = detection.get("language")
# Order-sensitive at every nesting level (#166): options are positional, so two
# equal schemas with different key orders must not share a group.
schema = _question_schema(ctx.questions)
for group in question_groups:
- if group["schema"] == schema and group["overrides"] == overrides:
+ if (group["schema"] == schema and group["overrides"] == overrides
+ and group["lang"] == lang_key):
group["items"].append((i, ctx))
break
else:
@@ -737,17 +750,35 @@ class Router(HookRegistry):
"questions": ctx.questions,
"schema": schema,
"overrides": overrides,
+ "lang": lang_key,
"items": [(i, ctx)],
})
for group in question_groups:
items = group["items"]
- batch_results = agent.predict_batch(
- [ctx.states[0] for _, ctx in items],
- group["questions"],
- batch_size=batch_size,
- **group["overrides"],
- )
+ batch_kwargs = dict(group["overrides"])
+ if group["lang"] is not None:
+ batch_kwargs["lang"] = group["lang"]
+ try:
+ batch_results = agent.predict_batch(
+ [ctx.states[0] for _, ctx in items],
+ group["questions"],
+ batch_size=batch_size,
+ **batch_kwargs,
+ )
+ except TypeError as e:
+ # Same tolerance `predict` has for an Agent-like object whose
+ # `predict_batch` predates the `lang` argument.
+ if batch_kwargs.get("lang") is not None and "unexpected keyword argument 'lang'" in str(e):
+ batch_kwargs.pop("lang")
+ batch_results = agent.predict_batch(
+ [ctx.states[0] for _, ctx in items],
+ group["questions"],
+ batch_size=batch_size,
+ **batch_kwargs,
+ )
+ else:
+ raise
if len(batch_results) != len(items):
raise RuntimeError(
diff --git a/tests/test_router_batch.py b/tests/test_router_batch.py
index 84a2694d..bfdb51a5 100644
--- a/tests/test_router_batch.py
+++ b/tests/test_router_batch.py
@@ -234,3 +234,61 @@ def test_equal_questions_with_different_option_order_score_separately(fake_agent
# separate agent calls, each carrying its own caller's option order
orders = [list(call[2]["intent"]["criteria"]) for call in calls]
assert orders == [["zulu", "alpha"], ["alpha", "zulu"]]
+
+
+def _lang_recording_router(monkeypatch, lang_temperatures):
+ """A Router over a fake agent that records the `lang` each batch call received."""
+ import laya.agent
+
+ calls = []
+
+ class Agent:
+ def __init__(self, repo, *, device, token, subfolder):
+ self.checkpoint = subfolder or "english"
+ if lang_temperatures is not None:
+ self.lang_temperatures = lang_temperatures
+
+ def predict_batch(self, states, questions, batch_size=None, **overrides):
+ calls.append({"n": len(states), "lang": overrides.get("lang")})
+ return [{"model": "fake", "answers": {}} for _ in states]
+
+ def system_one(self, state, questions, **overrides):
+ calls.append({"n": 1, "lang": overrides.get("lang")})
+ return {"model": "fake", "answers": {}}
+
+ monkeypatch.setattr(laya.agent, "Agent", Agent)
+ return Router(max_loaded=1, default="english"), calls
+
+
+def test_lang_reaches_the_agent_when_it_carries_lang_temperatures(monkeypatch):
+ """`predict` forwards the request's language so per-language temperatures apply; the batch
+ path forwarded only the token budgets, so the same request scored differently depending on
+ which entry point served it. The request is routed as German either way, which is what made
+ the difference hard to see."""
+ router, calls = _lang_recording_router(monkeypatch, {"de": [1.5, 1.5, 1.5]})
+ router.predict_batch([request("a", lang="de"), request("b", lang="de"), request("c", lang="fr")])
+ assert sorted(c["lang"] for c in calls) == ["de", "fr"]
+ # requests sharing a language still share one forward pass
+ assert {"n": 2, "lang": "de"} in calls
+ assert {"n": 1, "lang": "fr"} in calls
+
+
+def test_lang_is_not_added_to_the_group_key_without_lang_temperatures(monkeypatch):
+ """An agent with no per-language temperatures does not use `lang`, so naming it must not
+ split a group that shares one forward pass today. The explicit `model` keeps every request
+ on one checkpoint, so the only thing that could split the group is the lang key."""
+ router, calls = _lang_recording_router(monkeypatch, None)
+ router.predict_batch([request("a", model="english", lang="de"),
+ request("b", model="english", lang="fr"),
+ request("c", model="english")])
+ assert calls == [{"n": 3, "lang": None}]
+
+
+def test_predict_and_predict_batch_pass_the_same_lang(monkeypatch):
+ """The invariant that was violated: one request, either entry point, same language."""
+ router, calls = _lang_recording_router(monkeypatch, {"de": [1.5, 1.5, 1.5]})
+ router.predict("a", Q, model="english", lang="de")
+ via_predict = calls[-1]["lang"]
+ calls.clear()
+ router.predict_batch([request("a", model="english", lang="de")])
+ assert calls[-1]["lang"] == via_predict == "de"