Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(onnx): report state truncation in usage like Agent

Som Tripathi770e269795ef
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2c8b41e6..0312fa75 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -108,6 +108,7 @@ jobs:
python tests/test_structured_api.py
python tests/test_structured_docs.py
python tests/test_onnx_lang_parity.py
+ python tests/test_onnx_truncation_parity.py
python tests/test_onnx_batch.py
python tests/test_onnx_long.py
python tests/test_onnx_sort.py
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 500b9a03..952add48 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -46,6 +46,7 @@ jobs:
python tests/test_hooks.py
python tests/test_hooks_api.py
python tests/test_onnx_lang_parity.py
+ python tests/test_onnx_truncation_parity.py
python tests/test_onnx_batch.py
python tests/test_onnx_long.py
python tests/test_onnx_sort.py
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py
index c31778a9..49ef3a02 100644
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@ -540,9 +540,10 @@ class ONNXAgent(HookRegistry):
items = []
for qid in ids:
q = internal[qid]
- seq, markers, stats = build_sequence(
+ seq, markers, stats, state_stats = build_sequence(
self.tok, state, q, max_len, head_max_len,
truncate_left=truncate_left, state_ids=state_ids, return_stats=True,
+ return_truncation_stats=True,
)
n_opts = len(render_options(q))
if len(markers) != n_opts:
@@ -557,7 +558,8 @@ class ONNXAgent(HookRegistry):
"max_len=%d allows once head_max_len=%d is spent on them; lower head_max_len, "
"raise max_len, or use fewer options"
% (qid, n_opts, len(seq), max_len, head_max_len))
- items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]], "options": stats})
+ items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]], "options": stats,
+ "state_stats": state_stats})
return items
def _decode_answers(self, logits, act, items: List[Dict[str, Any]], ids: List[str],
@@ -680,7 +682,18 @@ class ONNXAgent(HookRegistry):
for index, items in zip(indices, per_state_items):
nrows = len(items)
n_tokens = int(att[row:row + nrows].sum())
- usage = {"input_tokens": n_tokens, "output_tokens": 0}
+ # Same truncation report as Agent.predict_batch (#174).
+ stats = [item["state_stats"] for item in items]
+ dropped = max(s["state_tokens_dropped"] for s in stats)
+ usage = {
+ "input_tokens": n_tokens,
+ "output_tokens": 0,
+ "state_tokens": stats[0]["state_tokens"],
+ # worst case: the questions share one state, not one head budget
+ "state_tokens_dropped": dropped,
+ "truncated": dropped > 0,
+ "truncated_questions": [qid for qid, s in zip(ids, stats) if s["truncated"]],
+ }
# Only when a question lost options to the head budget, as on the torch Agent.
collapsed = collapsed_options(ids, items)
if collapsed:
diff --git a/tests/test_onnx_truncation_parity.py b/tests/test_onnx_truncation_parity.py
new file mode 100644
index 00000000..9a0f463d
--- /dev/null
+++ b/tests/test_onnx_truncation_parity.py
@@ -0,0 +1,121 @@
+"""ONNXAgent must report state truncation in `usage` like the PyTorch Agent (#174).
+
+`Agent.predict_batch` adds `state_tokens`, `state_tokens_dropped`, `truncated` and
+`truncated_questions` to each result's usage, from the token budget `build_sequence` applied.
+A caller who swaps to `ONNXAgent` must get the same fields with the same meaning. This drives
+`_infer` with a stub session and a fake tokenizer (the technique in tests/test_onnx_lang_parity.py),
+so it needs neither onnxruntime nor a checkpoint.
+
+Run: python tests/test_onnx_truncation_parity.py
+"""
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import numpy as np # noqa: E402
+
+from laya.common import build_sequence # noqa: E402
+from laya.onnx_agent import ONNXAgent # noqa: E402
+
+PASS, FAIL = [], []
+
+
+def check(name, got, want):
+ if got == want:
+ PASS.append(name)
+ else:
+ FAIL.append("%s:\n got %r\n want %r" % (name, got, want))
+
+
+# ---------------------------------------------------------------- a fake tokenizer + stub session drive _infer
+class _FakeTok:
+ """One token per character, so a state's token count is its length."""
+ cls_token_id, sep_token_id, mask_token_id, pad_token_id = 1, 2, 3, 0
+ mask_token = "[M]"
+
+ def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):
+ ids = [10 + (ord(c) % 40) for c in text]
+ if truncation and max_length:
+ ids = ids[:max_length]
+ return {"input_ids": ids}
+
+
+class _StubSession:
+ def run(self, names, inputs):
+ n = inputs["input_ids"].shape[0]
+ return [np.zeros((n, 3), dtype=np.float32), np.zeros((n, 2), dtype=np.float32)]
+
+
+MAX_LEN, HEAD_MAX_LEN = 128, 96
+
+
+def _bare_onnx():
+ a = ONNXAgent.__new__(ONNXAgent)
+ a.model_id = "stub"
+ a.cfg = {"max_len": MAX_LEN, "head_max_len": HEAD_MAX_LEN}
+ a.tok = _FakeTok()
+ a.temperature = [1.0, 1.0, 1.0]
+ a.temperature_by_options = {}
+ a.lang_temperatures = {}
+ a.session = _StubSession()
+ return a
+
+
+QUESTIONS = {
+ "dept": {"type": "choice", "instructions": "Which team?",
+ "criteria": {"billing": "money", "support": "help", "sales": "buy"}},
+ "flag": {"type": "noul", "instructions": "Ok?"},
+}
+
+# The room each question leaves for the state: max_len minus the head and the closing [SEP].
+ROOM = {qid: MAX_LEN - len(build_sequence(_FakeTok(), "", ONNXAgent._to_internal(q), MAX_LEN, HEAD_MAX_LEN)[0])
+ for qid, q in QUESTIONS.items()}
+assert ROOM["flag"] < ROOM["dept"], ROOM # the cases below need two different budgets
+
+agent = _bare_onnx()
+
+# ---------------------------------------------------------------- empty questions: usage does not grow
+check("empty/usage is unchanged", agent._infer("some state", {})["usage"],
+ {"input_tokens": 0, "output_tokens": 0})
+
+# ---------------------------------------------------------------- a state that fits every question
+u = agent._infer("x" * ROOM["flag"], QUESTIONS)["usage"]
+check("fits/keys match the torch Agent", sorted(u),
+ sorted(["input_tokens", "output_tokens", "state_tokens", "state_tokens_dropped", "truncated",
+ "truncated_questions"]))
+check("fits/state_tokens", u["state_tokens"], ROOM["flag"])
+check("fits/nothing dropped", u["state_tokens_dropped"], 0)
+check("fits/truncated is False (a bool)", (u["truncated"], type(u["truncated"])), (False, bool))
+check("fits/no question listed", u["truncated_questions"], [])
+
+# ---------------------------------------------------------------- a state only the narrower question truncates
+n = ROOM["flag"] + 6
+u = agent._infer("x" * n, QUESTIONS)["usage"]
+check("one/state_tokens is the full state", u["state_tokens"], n)
+check("one/dropped is the worst question's", u["state_tokens_dropped"], n - ROOM["flag"])
+check("one/truncated", (u["truncated"], type(u["truncated"])), (True, bool))
+check("one/only the narrower question listed", u["truncated_questions"], ["flag"])
+
+# ---------------------------------------------------------------- a state every question truncates
+u = agent._infer("x" * 500, QUESTIONS)["usage"]
+check("all/state_tokens", u["state_tokens"], 500)
+check("all/dropped is the max over questions", u["state_tokens_dropped"], 500 - ROOM["flag"])
+check("all/every question listed, in order", u["truncated_questions"], ["dept", "flag"])
+
+# ---------------------------------------------------------------- a conversation list keeps its tail, same report
+convo = [{"role": "user", "content": "x" * 500}]
+u = agent._infer(convo, QUESTIONS)["usage"]
+check("list/truncated from the left is still reported", u["truncated"], True)
+
+# ---------------------------------------------------------------- predict_batch reports each state on its own
+res = agent.predict_batch(["x" * ROOM["flag"], "x" * (ROOM["flag"] + 6), "x" * 500], QUESTIONS)
+check("batch/per-state truncated", [r["usage"]["truncated"] for r in res], [False, True, True])
+check("batch/per-state questions", [r["usage"]["truncated_questions"] for r in res], [[], ["flag"], ["dept", "flag"]])
+check("batch/per-state state_tokens", [r["usage"]["state_tokens"] for r in res], [ROOM["flag"], ROOM["flag"] + 6, 500])
+
+
+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
+for f in FAIL:
+ print(" FAIL " + f)
+sys.exit(1 if FAIL else 0)