Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
perf(onnx): reuse question tokens across states in predict_batch
Hamidrezaee136b57e684
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.pyindex 27a91299..2b3a1b8c 100644--- a/laya/onnx_agent.py+++ b/laya/onnx_agent.py@@ -15,6 +15,7 @@ from laya.hooks import (from laya.revisions import resolve_revision, snapshot_revision, verify_digestsfrom laya.common import (QTYPES,+ _reuse_question_tokens,answer_confidence,build_sequence,collapsed_options,@@ -337,6 +338,7 @@ class ONNXAgent(HookRegistry):max_len=max_len, head_max_len=head_max_len,min_confidence=min_confidence)[0]+ @_reuse_question_tokensdef predict_batch(self, states: List[Union[str, dict, list]], questions: Dict[str, Dict[str, Any]],batch_size: Optional[int] = None, lang: Optional[str] = None,hooks=None, on_predict_start=None, on_predict_end=None,diff --git a/tests/test_confidence.py b/tests/test_confidence.pyindex f594c614..949db982 100644--- a/tests/test_confidence.py+++ b/tests/test_confidence.py@@ -434,7 +434,7 @@ check("export/laya re-exports the same tuple", laya.GATE_STATES, _confidence.GAT# `docs/structured.md` and `docs/questions-and-answers.md` each opened their confidence section with# "`confidence` is normalized entropy". Neither agent writes that for every type:# `Agent._decode_answers` (`laya/agent.py:1369-1398`) and `ONNXAgent._decode_answers`-# (`laya/onnx_agent.py:713-783`) put `confidence_from_probs(p, k)` = `1 - H(p) / log(k)` in the+# (`laya/onnx_agent.py:715-785`) put `confidence_from_probs(p, k)` = `1 - H(p) / log(k)` in the# `choice` and `score` answers and `max(p_true, 1 - p_true)` in the `noul` one. Each page printed a# `noul` in the very block its sentence introduces, so the sentence was falsifiable from the page# alone: the Q&A page's sample answer is `{"type": "noul", "noul": 0.8727, "confidence": 0.8727}`,diff --git a/tests/test_onnx_batch.py b/tests/test_onnx_batch.pyindex 97d9f58a..853d5b42 100644--- a/tests/test_onnx_batch.py+++ b/tests/test_onnx_batch.py@@ -362,6 +362,30 @@ check("single option/a run that already has two or more slots is sent as before"wide.session.marker_widths, [3])+# ---------------------------------------------------------------- question token reuse+# Agent.predict_batch caches question tokenizations across states (@_reuse_question_tokens);+# ONNXAgent.predict_batch shares that cache, so question instructions and options are encoded once+# per batch rather than once per state document.+tok_records = []+++class _TrackingTok(_FakeTok):+ def __call__(self, text, **kw):+ tok_records.append(text)+ return super().__call__(text, **kw)+++tracking_agent = _bare_onnx()+tracking_agent.tok = _TrackingTok()+tracking_agent.predict_batch(STATES, QUESTIONS)+dept_instruction = "choice question: Which team?"+check("reuse/question instruction encoded once across the batch",+ tok_records.count(dept_instruction), 1)+check("reuse/total tokenizations capped by reuse across states",+ len(tok_records), 7 + len(STATES))+++# --------------------------------------------------------------------------- reportprint("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL:diff --git a/tests/test_question_token_reuse.py b/tests/test_question_token_reuse.pyindex 9adf806e..4dc0ad15 100644--- a/tests/test_question_token_reuse.py+++ b/tests/test_question_token_reuse.py@@ -85,6 +85,34 @@ class QuestionTokenReuse(unittest.TestCase):for key in left:self.assertTrue(left[key].equal(right[key]), key)+ def test_onnx_agent_reuses_question_tokens_across_states(self):+ from laya.onnx_agent import ONNXAgent+ states = ["state %d" % i for i in range(10)]+ tok = Tokenizer()+ agent = ONNXAgent.__new__(ONNXAgent)+ agent.tok = tok+ agent.cfg = {"max_len": 512, "head_max_len": 192}+ agent.temperature = [1.0, 1.0, 1.0]+ agent.temperature_by_options = {}+ agent.lang_temperatures = {}+ agent.hooks = ()+ agent.hooks_raise = True+ agent.hooks_concurrent = True+ agent.hooks_timeout = None+ agent._hooks_lock = None+ agent._hooks_mutex = threading.Lock()++ class StubSession:+ def run(self, names, inputs):+ n = len(inputs["input_ids"])+ return [np.zeros((n, 2), dtype=np.float32), np.zeros((n, 2), dtype=np.float32)]++ agent.session = StubSession()+ tok.calls.clear()+ agent.predict_batch(states, QUESTIONS)+ # 10 states + 3 question encodes (1 instruction + 2 options)+ self.assertEqual(len(tok.calls), len(states) + 3)+def test_predict_long_uses_the_same_cache_for_all_windows(self):tok = Tokenizer()agent = agent_with(tok)