Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
perf(router): memoize question schema serialization in predict_batch
Hamidrezac907368e6275
diff --git a/laya/router.py b/laya/router.pyindex 7853ec9a..4113ba3f 100644--- a/laya/router.py+++ b/laya/router.py@@ -1713,6 +1713,8 @@ class Router(HookRegistry):# checkpoint group again on what the start hooks left, so a rewritten question# set or `ctx.max_len` only applies to its own request.question_groups: List[Dict[str, Any]] = []+ last_questions = None+ last_schema = Nonefor i, ctx in zip(indices, started):if ctx.results is not None:# A cache hit short-circuits inference; keep the `routing` key predict adds.@@ -1722,9 +1724,12 @@ class Router(HookRegistry):continue# Pass token-budget overrides only when set, as `predict` does, so an# Agent-like object that does not accept them still works.- overrides = {key: value for key, value in (("max_len", ctx.max_len),- ("head_max_len", ctx.head_max_len))- if value is not None}+ if ctx.max_len is None and ctx.head_max_len is None:+ overrides = {}+ else:+ overrides = {key: value for key, value in (("max_len", ctx.max_len),+ ("head_max_len", ctx.head_max_len))+ if value is not None}# `predict` forwards the language of the request so the agent can apply its# per-language temperatures; the batched path forwarded only the token# budgets, so the same request scored differently depending on the entry@@ -1739,7 +1744,12 @@ class Router(HookRegistry):lang_key = detection.get("language")# Order-sensitive at every nesting level (#166): options are positional, so two# equal schemas with different key orders must not share a group.- schema = _question_schema(ctx.questions)+ if ctx.questions is last_questions:+ schema = last_schema+ else:+ schema = _question_schema(ctx.questions)+ last_questions = ctx.questions+ last_schema = schemafor group in question_groups:if (group["schema"] == schema and group["overrides"] == overridesand group["lang"] == lang_key):diff --git a/tests/test_router_batch.py b/tests/test_router_batch.pyindex a5b3c38a..bd172878 100644--- a/tests/test_router_batch.py+++ b/tests/test_router_batch.py@@ -558,3 +558,32 @@ def test_sort_by_length_is_dropped_for_agents_that_predate_it(monkeypatch):assert len(results) == 2assert [c["sort"] for c in calls] == ["unsupported"] # one group, retry without the knobassert [r["routing"]["model"] for r in results] == ["english", "english"]+++def test_predict_batch_memoizes_identical_question_schema(monkeypatch):+ """Router.predict_batch memoizes question schema serialization across consecutive requests+ sharing the identical questions dictionary, avoiding redundant json.dumps() per item."""+ import laya.router+ calls = []+ real_schema = laya.router._question_schema++ def recording_schema(q):+ calls.append(q)+ return real_schema(q)++ monkeypatch.setattr(laya.router, "_question_schema", recording_schema)+ router, _ = _sort_recording_router(monkeypatch, accepts_sort=True)++ shared_q = {"intent": {"type": "noul", "instructions": "Urgent?"}}+ other_q = {"intent": {"type": "noul", "instructions": "Different?"}}++ requests = [+ {"state": "a", "questions": shared_q},+ {"state": "b", "questions": shared_q},+ {"state": "c", "questions": shared_q},+ {"state": "d", "questions": other_q},+ ]+ results = router.predict_batch(requests)+ assert len(results) == 4+ # Exactly 2 schema serializations instead of 4+ assert len(calls) == 2