Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

perf(onnx): reuse question tokens across states in predict_batch

Hamidrezaee136b57e684
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py
index 27a91299..2b3a1b8c 100644
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@ -15,6 +15,7 @@ from laya.hooks import (
from laya.revisions import resolve_revision, snapshot_revision, verify_digests
from laya.common import (
QTYPES,
+ _reuse_question_tokens,
answer_confidence,
build_sequence,
collapsed_options,
@@ -337,6 +338,7 @@ class ONNXAgent(HookRegistry):
max_len=max_len, head_max_len=head_max_len,
min_confidence=min_confidence)[0]
+ @_reuse_question_tokens
def predict_batch(self, states: List[Union[str, dict, list]], questions: Dict[str, Dict[str, Any]],
batch_size: Optional[int] = None, lang: Optional[str] = None,
hooks=None, on_predict_start=None, on_predict_end=None,
diff --git a/tests/test_confidence.py b/tests/test_confidence.py
index f594c614..949db982 100644
--- a/tests/test_confidence.py
+++ b/tests/test_confidence.py
@@ -434,7 +434,7 @@ check("export/laya re-exports the same tuple", laya.GATE_STATES, _confidence.GAT
# `docs/structured.md` and `docs/questions-and-answers.md` each opened their confidence section with
# "`confidence` is normalized entropy". Neither agent writes that for every type:
# `Agent._decode_answers` (`laya/agent.py:1369-1398`) and `ONNXAgent._decode_answers`
-# (`laya/onnx_agent.py:713-783`) put `confidence_from_probs(p, k)` = `1 - H(p) / log(k)` in the
+# (`laya/onnx_agent.py:715-785`) put `confidence_from_probs(p, k)` = `1 - H(p) / log(k)` in the
# `choice` and `score` answers and `max(p_true, 1 - p_true)` in the `noul` one. Each page printed a
# `noul` in the very block its sentence introduces, so the sentence was falsifiable from the page
# alone: the Q&A page's sample answer is `{"type": "noul", "noul": 0.8727, "confidence": 0.8727}`,
diff --git a/tests/test_onnx_batch.py b/tests/test_onnx_batch.py
index 97d9f58a..853d5b42 100644
--- a/tests/test_onnx_batch.py
+++ b/tests/test_onnx_batch.py
@@ -362,6 +362,30 @@ check("single option/a run that already has two or more slots is sent as before"
wide.session.marker_widths, [3])
+# ---------------------------------------------------------------- question token reuse
+# Agent.predict_batch caches question tokenizations across states (@_reuse_question_tokens);
+# ONNXAgent.predict_batch shares that cache, so question instructions and options are encoded once
+# per batch rather than once per state document.
+tok_records = []
+
+
+class _TrackingTok(_FakeTok):
+ def __call__(self, text, **kw):
+ tok_records.append(text)
+ return super().__call__(text, **kw)
+
+
+tracking_agent = _bare_onnx()
+tracking_agent.tok = _TrackingTok()
+tracking_agent.predict_batch(STATES, QUESTIONS)
+dept_instruction = "choice question: Which team?"
+check("reuse/question instruction encoded once across the batch",
+ tok_records.count(dept_instruction), 1)
+check("reuse/total tokenizations capped by reuse across states",
+ len(tok_records), 7 + len(STATES))
+
+
+
# --------------------------------------------------------------------------- report
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
diff --git a/tests/test_question_token_reuse.py b/tests/test_question_token_reuse.py
index 9adf806e..4dc0ad15 100644
--- a/tests/test_question_token_reuse.py
+++ b/tests/test_question_token_reuse.py
@@ -85,6 +85,34 @@ class QuestionTokenReuse(unittest.TestCase):
for key in left:
self.assertTrue(left[key].equal(right[key]), key)
+ def test_onnx_agent_reuses_question_tokens_across_states(self):
+ from laya.onnx_agent import ONNXAgent
+ states = ["state %d" % i for i in range(10)]
+ tok = Tokenizer()
+ agent = ONNXAgent.__new__(ONNXAgent)
+ agent.tok = tok
+ agent.cfg = {"max_len": 512, "head_max_len": 192}
+ agent.temperature = [1.0, 1.0, 1.0]
+ agent.temperature_by_options = {}
+ agent.lang_temperatures = {}
+ agent.hooks = ()
+ agent.hooks_raise = True
+ agent.hooks_concurrent = True
+ agent.hooks_timeout = None
+ agent._hooks_lock = None
+ agent._hooks_mutex = threading.Lock()
+
+ class StubSession:
+ def run(self, names, inputs):
+ n = len(inputs["input_ids"])
+ return [np.zeros((n, 2), dtype=np.float32), np.zeros((n, 2), dtype=np.float32)]
+
+ agent.session = StubSession()
+ tok.calls.clear()
+ agent.predict_batch(states, QUESTIONS)
+ # 10 states + 3 question encodes (1 instruction + 2 options)
+ self.assertEqual(len(tok.calls), len(states) + 3)
+
def test_predict_long_uses_the_same_cache_for_all_windows(self):
tok = Tokenizer()
agent = agent_with(tok)