Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(onnx): report state truncation in usage like Agent
Som Tripathi770e269795ef
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 2c8b41e6..0312fa75 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -108,6 +108,7 @@ jobs:python tests/test_structured_api.pypython tests/test_structured_docs.pypython tests/test_onnx_lang_parity.py+ python tests/test_onnx_truncation_parity.pypython tests/test_onnx_batch.pypython tests/test_onnx_long.pypython tests/test_onnx_sort.pydiff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 500b9a03..952add48 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -46,6 +46,7 @@ jobs:python tests/test_hooks.pypython tests/test_hooks_api.pypython tests/test_onnx_lang_parity.py+ python tests/test_onnx_truncation_parity.pypython tests/test_onnx_batch.pypython tests/test_onnx_long.pypython tests/test_onnx_sort.pydiff --git a/laya/onnx_agent.py b/laya/onnx_agent.pyindex c31778a9..49ef3a02 100644--- a/laya/onnx_agent.py+++ b/laya/onnx_agent.py@@ -540,9 +540,10 @@ class ONNXAgent(HookRegistry):items = []for qid in ids:q = internal[qid]- seq, markers, stats = build_sequence(+ seq, markers, stats, state_stats = build_sequence(self.tok, state, q, max_len, head_max_len,truncate_left=truncate_left, state_ids=state_ids, return_stats=True,+ return_truncation_stats=True,)n_opts = len(render_options(q))if len(markers) != n_opts:@@ -557,7 +558,8 @@ class ONNXAgent(HookRegistry):"max_len=%d allows once head_max_len=%d is spent on them; lower head_max_len, ""raise max_len, or use fewer options"% (qid, n_opts, len(seq), max_len, head_max_len))- items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]], "options": stats})+ items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]], "options": stats,+ "state_stats": state_stats})return itemsdef _decode_answers(self, logits, act, items: List[Dict[str, Any]], ids: List[str],@@ -680,7 +682,18 @@ class ONNXAgent(HookRegistry):for index, items in zip(indices, per_state_items):nrows = len(items)n_tokens = int(att[row:row + nrows].sum())- usage = {"input_tokens": n_tokens, "output_tokens": 0}+ # Same truncation report as Agent.predict_batch (#174).+ stats = [item["state_stats"] for item in items]+ dropped = max(s["state_tokens_dropped"] for s in stats)+ usage = {+ "input_tokens": n_tokens,+ "output_tokens": 0,+ "state_tokens": stats[0]["state_tokens"],+ # worst case: the questions share one state, not one head budget+ "state_tokens_dropped": dropped,+ "truncated": dropped > 0,+ "truncated_questions": [qid for qid, s in zip(ids, stats) if s["truncated"]],+ }# Only when a question lost options to the head budget, as on the torch Agent.collapsed = collapsed_options(ids, items)if collapsed:diff --git a/tests/test_onnx_truncation_parity.py b/tests/test_onnx_truncation_parity.pynew file mode 100644index 00000000..9a0f463d--- /dev/null+++ b/tests/test_onnx_truncation_parity.py@@ -0,0 +1,121 @@+"""ONNXAgent must report state truncation in `usage` like the PyTorch Agent (#174).++`Agent.predict_batch` adds `state_tokens`, `state_tokens_dropped`, `truncated` and+`truncated_questions` to each result's usage, from the token budget `build_sequence` applied.+A caller who swaps to `ONNXAgent` must get the same fields with the same meaning. This drives+`_infer` with a stub session and a fake tokenizer (the technique in tests/test_onnx_lang_parity.py),+so it needs neither onnxruntime nor a checkpoint.++Run: python tests/test_onnx_truncation_parity.py+"""+import os+import sys++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++import numpy as np # noqa: E402++from laya.common import build_sequence # noqa: E402+from laya.onnx_agent import ONNXAgent # noqa: E402++PASS, FAIL = [], []+++def check(name, got, want):+ if got == want:+ PASS.append(name)+ else:+ FAIL.append("%s:\n got %r\n want %r" % (name, got, want))+++# ---------------------------------------------------------------- a fake tokenizer + stub session drive _infer+class _FakeTok:+ """One token per character, so a state's token count is its length."""+ cls_token_id, sep_token_id, mask_token_id, pad_token_id = 1, 2, 3, 0+ mask_token = "[M]"++ def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):+ ids = [10 + (ord(c) % 40) for c in text]+ if truncation and max_length:+ ids = ids[:max_length]+ return {"input_ids": ids}+++class _StubSession:+ def run(self, names, inputs):+ n = inputs["input_ids"].shape[0]+ return [np.zeros((n, 3), dtype=np.float32), np.zeros((n, 2), dtype=np.float32)]+++MAX_LEN, HEAD_MAX_LEN = 128, 96+++def _bare_onnx():+ a = ONNXAgent.__new__(ONNXAgent)+ a.model_id = "stub"+ a.cfg = {"max_len": MAX_LEN, "head_max_len": HEAD_MAX_LEN}+ a.tok = _FakeTok()+ a.temperature = [1.0, 1.0, 1.0]+ a.temperature_by_options = {}+ a.lang_temperatures = {}+ a.session = _StubSession()+ return a+++QUESTIONS = {+ "dept": {"type": "choice", "instructions": "Which team?",+ "criteria": {"billing": "money", "support": "help", "sales": "buy"}},+ "flag": {"type": "noul", "instructions": "Ok?"},+}++# The room each question leaves for the state: max_len minus the head and the closing [SEP].+ROOM = {qid: MAX_LEN - len(build_sequence(_FakeTok(), "", ONNXAgent._to_internal(q), MAX_LEN, HEAD_MAX_LEN)[0])+ for qid, q in QUESTIONS.items()}+assert ROOM["flag"] < ROOM["dept"], ROOM # the cases below need two different budgets++agent = _bare_onnx()++# ---------------------------------------------------------------- empty questions: usage does not grow+check("empty/usage is unchanged", agent._infer("some state", {})["usage"],+ {"input_tokens": 0, "output_tokens": 0})++# ---------------------------------------------------------------- a state that fits every question+u = agent._infer("x" * ROOM["flag"], QUESTIONS)["usage"]+check("fits/keys match the torch Agent", sorted(u),+ sorted(["input_tokens", "output_tokens", "state_tokens", "state_tokens_dropped", "truncated",+ "truncated_questions"]))+check("fits/state_tokens", u["state_tokens"], ROOM["flag"])+check("fits/nothing dropped", u["state_tokens_dropped"], 0)+check("fits/truncated is False (a bool)", (u["truncated"], type(u["truncated"])), (False, bool))+check("fits/no question listed", u["truncated_questions"], [])++# ---------------------------------------------------------------- a state only the narrower question truncates+n = ROOM["flag"] + 6+u = agent._infer("x" * n, QUESTIONS)["usage"]+check("one/state_tokens is the full state", u["state_tokens"], n)+check("one/dropped is the worst question's", u["state_tokens_dropped"], n - ROOM["flag"])+check("one/truncated", (u["truncated"], type(u["truncated"])), (True, bool))+check("one/only the narrower question listed", u["truncated_questions"], ["flag"])++# ---------------------------------------------------------------- a state every question truncates+u = agent._infer("x" * 500, QUESTIONS)["usage"]+check("all/state_tokens", u["state_tokens"], 500)+check("all/dropped is the max over questions", u["state_tokens_dropped"], 500 - ROOM["flag"])+check("all/every question listed, in order", u["truncated_questions"], ["dept", "flag"])++# ---------------------------------------------------------------- a conversation list keeps its tail, same report+convo = [{"role": "user", "content": "x" * 500}]+u = agent._infer(convo, QUESTIONS)["usage"]+check("list/truncated from the left is still reported", u["truncated"], True)++# ---------------------------------------------------------------- predict_batch reports each state on its own+res = agent.predict_batch(["x" * ROOM["flag"], "x" * (ROOM["flag"] + 6), "x" * 500], QUESTIONS)+check("batch/per-state truncated", [r["usage"]["truncated"] for r in res], [False, True, True])+check("batch/per-state questions", [r["usage"]["truncated_questions"] for r in res], [[], ["flag"], ["dept", "flag"]])+check("batch/per-state state_tokens", [r["usage"]["state_tokens"] for r in res], [ROOM["flag"], ROOM["flag"] + 6, 500])+++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))+for f in FAIL:+ print(" FAIL " + f)+sys.exit(1 if FAIL else 0)