Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

perf(router): memoize question schema serialization in predict_batch

Hamidrezac907368e6275
diff --git a/laya/router.py b/laya/router.py
index 7853ec9a..4113ba3f 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -1713,6 +1713,8 @@ class Router(HookRegistry):
# checkpoint group again on what the start hooks left, so a rewritten question
# set or `ctx.max_len` only applies to its own request.
question_groups: List[Dict[str, Any]] = []
+ last_questions = None
+ last_schema = None
for i, ctx in zip(indices, started):
if ctx.results is not None:
# A cache hit short-circuits inference; keep the `routing` key predict adds.
@@ -1722,9 +1724,12 @@ class Router(HookRegistry):
continue
# Pass token-budget overrides only when set, as `predict` does, so an
# Agent-like object that does not accept them still works.
- overrides = {key: value for key, value in (("max_len", ctx.max_len),
- ("head_max_len", ctx.head_max_len))
- if value is not None}
+ if ctx.max_len is None and ctx.head_max_len is None:
+ overrides = {}
+ else:
+ overrides = {key: value for key, value in (("max_len", ctx.max_len),
+ ("head_max_len", ctx.head_max_len))
+ if value is not None}
# `predict` forwards the language of the request so the agent can apply its
# per-language temperatures; the batched path forwarded only the token
# budgets, so the same request scored differently depending on the entry
@@ -1739,7 +1744,12 @@ class Router(HookRegistry):
lang_key = detection.get("language")
# Order-sensitive at every nesting level (#166): options are positional, so two
# equal schemas with different key orders must not share a group.
- schema = _question_schema(ctx.questions)
+ if ctx.questions is last_questions:
+ schema = last_schema
+ else:
+ schema = _question_schema(ctx.questions)
+ last_questions = ctx.questions
+ last_schema = schema
for group in question_groups:
if (group["schema"] == schema and group["overrides"] == overrides
and group["lang"] == lang_key):
diff --git a/tests/test_router_batch.py b/tests/test_router_batch.py
index a5b3c38a..bd172878 100644
--- a/tests/test_router_batch.py
+++ b/tests/test_router_batch.py
@@ -558,3 +558,32 @@ def test_sort_by_length_is_dropped_for_agents_that_predate_it(monkeypatch):
assert len(results) == 2
assert [c["sort"] for c in calls] == ["unsupported"] # one group, retry without the knob
assert [r["routing"]["model"] for r in results] == ["english", "english"]
+
+
+def test_predict_batch_memoizes_identical_question_schema(monkeypatch):
+ """Router.predict_batch memoizes question schema serialization across consecutive requests
+ sharing the identical questions dictionary, avoiding redundant json.dumps() per item."""
+ import laya.router
+ calls = []
+ real_schema = laya.router._question_schema
+
+ def recording_schema(q):
+ calls.append(q)
+ return real_schema(q)
+
+ monkeypatch.setattr(laya.router, "_question_schema", recording_schema)
+ router, _ = _sort_recording_router(monkeypatch, accepts_sort=True)
+
+ shared_q = {"intent": {"type": "noul", "instructions": "Urgent?"}}
+ other_q = {"intent": {"type": "noul", "instructions": "Different?"}}
+
+ requests = [
+ {"state": "a", "questions": shared_q},
+ {"state": "b", "questions": shared_q},
+ {"state": "c", "questions": shared_q},
+ {"state": "d", "questions": other_q},
+ ]
+ results = router.predict_batch(requests)
+ assert len(results) == 4
+ # Exactly 2 schema serializations instead of 4
+ assert len(calls) == 2