Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(onnx): port predict_batch sort_by_length to ONNXAgent

Aashisha6641f364ecd
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ca4dd36b..11e76f06 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -106,6 +106,7 @@ jobs:
python tests/test_onnx_lang_parity.py
python tests/test_onnx_batch.py
python tests/test_onnx_long.py
+ python tests/test_onnx_sort.py
python tests/test_download.py
python tests/test_shortlist.py
python tests/test_decision_model.py
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 8884e017..2747d00c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -45,6 +45,7 @@ jobs:
python tests/test_onnx_lang_parity.py
python tests/test_onnx_batch.py
python tests/test_onnx_long.py
+ python tests/test_onnx_sort.py
python tests/test_email.py
python tests/test_download.py
python tests/test_shortlist.py
diff --git a/README.md b/README.md
index 6c6965ad..0592086c 100644
--- a/README.md
+++ b/README.md
@@ -550,7 +550,8 @@ It looks ahead at most eight batches and reuses the encoded rows for sorting. Th
CPU memory for tokenized inputs, and takes effect only when `1 < batch_size < len(states)`.
Benchmark it on your workload and backend: uniform lengths offer little benefit, and changed
batch shapes can cause small floating-point differences, including near decision thresholds.
-Hooks still see states and final results in input order. The option is available on `Agent`.
+Hooks still see states and final results in input order. The option is available on `Agent` and
+`ONNXAgent`.
Results are aligned with `states` by index and identical in shape to `predict`. Changing batch
shapes can introduce floating-point differences on CPU and GPU; check decision thresholds on
@@ -560,9 +561,12 @@ batched (measured ~9–10×). On CPU, increasing batch size alone may not speed
length grouping can help by reducing the padded work in a mixed-length workload. See the
[CPU measurements and reproduction commands](research/README.md#length-batching).
-`ONNXAgent.predict_batch(states, questions, batch_size=...)` has the same contract, backed by one
-ONNX Runtime session run per chunk, so an ONNX deployment gets the same batch API and the same
-result shape; `sort_by_length` is not ported there yet.
+`ONNXAgent.predict_batch(states, questions, batch_size=..., sort_by_length=...)` has the same
+contract, backed by one ONNX Runtime session run per chunk, so an ONNX deployment gets the same
+batch API, the same result shape, and the same length grouping. Measured on the English checkpoint
+(flat fp32 export, Apple M1, 80 support tickets alternating short and ~8x-longer documents,
+`batch_size=4`, best of 3): wall clock 53.7 s unsorted → 36.3 s sorted (**~1.48x**), with 0/80
+decision changes and max probability drift 0.0 across queue/noul/score.
### Long documents: `predict_long`
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py
index 7d145c31..f53ad4c5 100644
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@ -274,6 +274,7 @@ class ONNXAgent(HookRegistry):
hooks_timeout: Optional[float] = None,
max_len: Optional[int] = None,
head_max_len: Optional[int] = None,
+ sort_by_length: bool = False,
min_confidence: Optional[float] = None) -> List[Dict[str, Any]]:
"""Evaluate the same questions over many states, sharing ONNX Runtime session runs.
@@ -298,6 +299,11 @@ class ONNXAgent(HookRegistry):
hooks_timeout: Override the agent's `hooks_timeout` for this call.
max_len: Override the config's `max_len` for this call.
head_max_len: Override the config's `head_max_len` for this call.
+ sort_by_length: Group similarly sized encoded states within windows of eight batches
+ to reduce padding, exactly as `Agent.predict_batch` does. Requires an explicit
+ `batch_size` greater than one and smaller than the number of states; otherwise
+ it has no effect. Results retain input order. Changing batch shapes can
+ slightly change floating-point predictions near decision thresholds.
min_confidence: Opt-in abstention threshold on `answer_confidence` (#361); answers
below it are returned flagged with `low_confidence: True`.
@@ -331,7 +337,8 @@ class ONNXAgent(HookRegistry):
if ctx.head_max_len is not None:
overrides["head_max_len"] = ctx.head_max_len
ctx.results = self._infer_batch(states, questions, lang=lang,
- batch_size=batch_size, **overrides)
+ batch_size=batch_size,
+ sort_by_length=sort_by_length, **overrides)
except BaseException as exc:
ctx.error = exc
try:
@@ -541,8 +548,14 @@ class ONNXAgent(HookRegistry):
def _infer_batch(self, states: List[Union[str, dict, list]], questions: Dict[str, Dict[str, Any]],
max_len: Optional[int] = None, head_max_len: Optional[int] = None,
lang: Optional[str] = None,
- batch_size: Optional[int] = None) -> List[Dict[str, Any]]:
- """Validate once, then encode, collate, run and decode in chunks of `batch_size` states."""
+ batch_size: Optional[int] = None,
+ sort_by_length: bool = False) -> List[Dict[str, Any]]:
+ """Validate once, then encode, collate, run and decode in chunks of `batch_size` states.
+
+ With `sort_by_length`, chunks are formed from length-sorted states inside a lookahead
+ window of eight batches (the same contract as `Agent.predict_batch`), so a batch pads to
+ a shorter maximum; results are returned in input order either way.
+ """
from .agent import Agent as _Agent
ids = list(questions.keys())
@@ -557,41 +570,53 @@ class ONNXAgent(HookRegistry):
chunk = batch_size if (batch_size and batch_size > 0) else len(states)
results: List[Dict[str, Any]] = []
- for start in range(0, len(states), chunk):
- part = states[start:start + chunk]
+ reorder = sort_by_length and 1 < chunk < len(states)
+ # Bound the tokenized lookahead independently of the input size (same as the torch Agent):
+ # sort within windows of eight batches, using each state's longest encoded question row.
+ window = chunk * 8 if reorder else chunk
+ for start in range(0, len(states), window):
+ part = states[start:start + window]
encoded = [self._encode_state(st, ids, internal, max_len, head_max_len) for st in part]
+ order = list(range(len(encoded)))
+ if reorder:
+ order.sort(key=lambda i: max(len(item["ids"]) for item in encoded[i]))
+ window_results: List[Optional[Dict[str, Any]]] = [None] * len(encoded)
+ for offset in range(0, len(order), chunk):
+ indices = order[offset:offset + chunk]
+ per_state_items = [encoded[i] for i in indices]
+
+ b = collate_items(per_state_items, self.tok.pad_token_id)
+
+ # Prepare ONNX inputs as numpy arrays
+ ort_inputs = {
+ "input_ids": b["input_ids"].numpy().astype(np.int64),
+ "attention_mask": b["attention_mask"].numpy().astype(np.int64),
+ "marker_pos": b["marker_pos"].numpy().astype(np.int64),
+ "marker_mask": b["marker_mask"].numpy().astype(bool),
+ "qtype": b["qtype"].numpy().astype(np.int64),
+ }
- b = collate_items(encoded, self.tok.pad_token_id)
-
- # Prepare ONNX inputs as numpy arrays
- ort_inputs = {
- "input_ids": b["input_ids"].numpy().astype(np.int64),
- "attention_mask": b["attention_mask"].numpy().astype(np.int64),
- "marker_pos": b["marker_pos"].numpy().astype(np.int64),
- "marker_mask": b["marker_mask"].numpy().astype(bool),
- "qtype": b["qtype"].numpy().astype(np.int64),
- }
-
- # Run ONNX inference
- ort_outs = self.session.run(["logits", "act_logits"], ort_inputs)
- logits = ort_outs[0]
- act_logits = ort_outs[1]
-
- # Compute softmax for actions manually in numpy
- act_exp = np.exp(act_logits - np.max(act_logits, axis=-1, keepdims=True))
- act = act_exp / np.sum(act_exp, axis=-1, keepdims=True)
- att = b["attention_mask"]
-
- row = 0
- for index, items in enumerate(encoded):
- nrows = len(items)
- n_tokens = int(att[row:row + nrows].sum())
- results.append({
- "model": "laya-rl-agent-onnx",
- "answers": self._decode_answers(logits, act, items, ids, internal, row, lang=lang),
- "usage": {"input_tokens": n_tokens, "output_tokens": 0},
- })
- row += nrows
+ # Run ONNX inference
+ ort_outs = self.session.run(["logits", "act_logits"], ort_inputs)
+ logits = ort_outs[0]
+ act_logits = ort_outs[1]
+
+ # Compute softmax for actions manually in numpy
+ act_exp = np.exp(act_logits - np.max(act_logits, axis=-1, keepdims=True))
+ act = act_exp / np.sum(act_exp, axis=-1, keepdims=True)
+ att = b["attention_mask"]
+
+ row = 0
+ for index, items in zip(indices, per_state_items):
+ nrows = len(items)
+ n_tokens = int(att[row:row + nrows].sum())
+ window_results[index] = {
+ "model": "laya-rl-agent-onnx",
+ "answers": self._decode_answers(logits, act, items, ids, internal, row, lang=lang),
+ "usage": {"input_tokens": n_tokens, "output_tokens": 0},
+ }
+ row += nrows
+ results.extend(window_results)
return results
def decide(self, state: Union[str, dict, list], schema: Any = None, *,
diff --git a/tests/test_onnx_batch.py b/tests/test_onnx_batch.py
index 9973c498..77686ec6 100644
--- a/tests/test_onnx_batch.py
+++ b/tests/test_onnx_batch.py
@@ -127,8 +127,8 @@ _shared = ["batch_size", "lang", "hooks", "on_predict_start", "on_predict_end",
"max_len", "head_max_len", "min_confidence"]
check("contract/parameter names match Agent.predict_batch",
[p for p in _shared if p not in _onnx_so], [])
-check("contract/only Agent-only knobs are missing",
- sorted(set(_agent_so) - set(_onnx_so)), ["sort_by_length"])
+check("contract/the Agent's full knob set is now ported",
+ sorted(set(_agent_so) - set(_onnx_so)), [])
# ---------------------------------------------------------------- batched == sequential
diff --git a/tests/test_onnx_sort.py b/tests/test_onnx_sort.py
new file mode 100644
index 00000000..01970935
--- /dev/null
+++ b/tests/test_onnx_sort.py
@@ -0,0 +1,172 @@
+"""ONNXAgent.predict_batch(sort_by_length=True): the length-grouping knob, ported from the Agent.
+
+`Agent.predict_batch` sorts encoded states inside windows of eight batches so each shared run
+pads to a shorter maximum (#294). The port must keep three properties: results stay aligned with
+the input order, the knob is a no-op outside `1 < batch_size < len(states)`, and grouping really
+does reduce padded rows x width. No onnxruntime and no checkpoint: the same stub-session
+technique as tests/test_onnx_batch.py, extended to record each run's padded width.
+
+Run: python tests/test_onnx_sort.py
+"""
+import os
+import sys
+import threading
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import numpy as np # noqa: E402
+
+from laya.onnx_agent import ONNXAgent # noqa: E402
+
+PASS, FAIL = [], []
+
+
+def check(name, got, want):
+ if got == want:
+ PASS.append(name)
+ else:
+ FAIL.append("%s:\n got %r\n want %r" % (name, got, want))
+
+
+def check_true(name, cond, detail=""):
+ if cond:
+ PASS.append(name)
+ else:
+ FAIL.append("%s %s" % (name, detail))
+
+
+class _FakeTok:
+ cls_token_id, sep_token_id, mask_token_id, pad_token_id = 1, 2, 3, 0
+ mask_token = "[M]"
+
+ def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):
+ ids = [10 + (ord(c) % 40) for c in text]
+ if truncation and max_length:
+ ids = ids[:max_length]
+ return {"input_ids": ids}
+
+
+class _StubSession:
+ """Records (rows, padded width) per run; row logits derive from the row's own tokens."""
+
+ def __init__(self):
+ self.calls = [] # [(rows, width), ...]
+
+ def run(self, names, inputs):
+ ids = inputs["input_ids"]
+ self.calls.append((len(ids), int(ids.shape[1])))
+ weight = 1.0 + (ids.sum(axis=1) % 4)
+ logits = np.stack([weight, np.ones_like(weight)], axis=1).astype(np.float32)
+ act = np.tile(np.array([[0.25, 0.75]], dtype=np.float32), (len(ids), 1))
+ return [logits, act]
+
+ @property
+ def padded_cells(self):
+ return sum(rows * width for rows, width in self.calls)
+
+
+def _bare_onnx():
+ a = ONNXAgent.__new__(ONNXAgent) # skip __init__: no onnxruntime, no checkpoint
+ a.model_id = "stub"
+ a.cfg = {"max_len": 128, "head_max_len": 32}
+ a.tok = _FakeTok()
+ a.temperature = [1.0, 1.0, 1.0]
+ a.temperature_by_options = {}
+ a.session = _StubSession()
+ a.hooks = ()
+ a.hooks_raise = True
+ a.hooks_concurrent = True
+ a.hooks_timeout = None
+ a._hooks_lock = None
+ a._hooks_mutex = threading.Lock()
+ a.lang_temperatures = {}
+ return a
+
+
+QUESTIONS = {
+ "dept": {"type": "choice", "instructions": "Which team?",
+ "criteria": {"billing": "money", "support": "help", "sales": "buy"}},
+ "flag": {"type": "noul", "instructions": "Urgent?"},
+}
+ROWS_PER_STATE = 2 # two questions -> two head rows per state
+
+# 12 states alternating short/long, characters cycling so adjacent states differ in mod-4
+# weight: a row decoded for the wrong state changes the answer and fails the order checks.
+_LETTERS = "abcd"
+STATES = [(_LETTERS[i % 4]) * (4 if i % 2 == 0 else 40) for i in range(12)]
+
+
+def results_of(sort, batch_size=4, states=STATES):
+ agent = _bare_onnx()
+ res = agent.predict_batch(states, QUESTIONS, batch_size=batch_size, sort_by_length=sort)
+ return res, agent.session
+
+
+# ---------------------------------------------------------------- no-op conditions
+base, base_sess = results_of(False)
+for bs, runs in ((None, 1), (1, len(STATES)), (len(STATES), 1)):
+ agent = _bare_onnx()
+ got = agent.predict_batch(STATES, QUESTIONS, batch_size=bs, sort_by_length=True)
+ check("noop/batch_size=%r: sort_by_length changes nothing" % bs, got, base)
+ check("noop/batch_size=%r: session runs unchanged by the knob" % bs,
+ len(agent.session.calls), runs)
+
+# default is off
+agent = _bare_onnx()
+check("default/sort_by_length defaults to the unsorted behaviour",
+ agent.predict_batch(STATES, QUESTIONS, batch_size=4), base)
+
+# ---------------------------------------------------------------- order and parity
+sorted_res, sorted_sess = results_of(True)
+check("order/results stay aligned with the input order", sorted_res, base)
+check("shape/same number of runs sorted vs unsorted",
+ [r for r, _ in sorted_sess.calls], [r for r, _ in base_sess.calls])
+
+# ---------------------------------------------------------------- padding actually shrinks
+check_true("padding/mixed lengths are really present unsorted (every batch pads to the long width)",
+ all(w == base_sess.calls[0][1] > 40 for _, w in base_sess.calls),
+ base_sess.calls)
+check_true("padding/sorting narrows batches without widening any",
+ min(w for _, w in sorted_sess.calls) < min(w for _, w in base_sess.calls)
+ and max(w for _, w in sorted_sess.calls) <= max(w for _, w in base_sess.calls)
+ and sum(w for _, w in sorted_sess.calls) < sum(w for _, w in base_sess.calls),
+ (base_sess.calls, sorted_sess.calls))
+check_true("padding/total padded cells drop with sorting",
+ sorted_sess.padded_cells < base_sess.padded_cells,
+ (sorted_sess.padded_cells, base_sess.padded_cells))
+# shortest states first: the first sorted run pads to the short length only
+check("padding/first sorted batch holds the short group (4 states x 2 rows)",
+ sorted_sess.calls[0], (ROWS_PER_STATE * 4, sorted_sess.calls[0][1]))
+check_true("padding/first sorted batch is narrower than the first unsorted one",
+ sorted_sess.calls[0][1] < base_sess.calls[0][1],
+ (sorted_sess.calls[0], base_sess.calls[0]))
+
+# ---------------------------------------------------------------- lookahead window
+# 40 states at batch_size=4 sorts within windows of 4*8=32, then a second window of 8.
+many = [(_LETTERS[i % 4]) * (4 + 4 * (i % 5)) for i in range(40)]
+agent = _bare_onnx()
+res = agent.predict_batch(many, QUESTIONS, batch_size=4, sort_by_length=True)
+check("window/40 states at batch_size=4 -> 10 runs", len(agent.session.calls), 10)
+sequential = [_bare_onnx().system_one(s, QUESTIONS) for s in many]
+check("window/results still match per-state calls across window boundaries", res, sequential)
+check("window/within a window, lengths are non-decreasing across runs",
+ [w for _, w in agent.session.calls[:8]], sorted(w for _, w in agent.session.calls[:8]))
+
+# ---------------------------------------------------------------- hooks still see input order
+seen = []
+
+
+def _end(ctx):
+ seen.append([r["answers"]["dept"]["probabilities"]["billing"] for r in ctx.results])
+
+
+agent = _bare_onnx()
+agent.predict_batch(STATES, QUESTIONS, batch_size=4, sort_by_length=True, on_predict_end=_end)
+check("hooks/end hook sees results in input order",
+ seen[0], [r["answers"]["dept"]["probabilities"]["billing"] for r in base])
+
+# ---------------------------------------------------------------- report
+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
+for f in FAIL:
+ print(" FAIL " + f)
+sys.exit(1 if FAIL else 0)