Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

17.6 KB
Raw Blame
"""ONNXAgent.predict_batch: the batch path the PyTorch Agent has and ONNX did not (#323).

`Router.predict_batch` calls `agent.predict_batch` unconditionally, so an ONNX-backed Router raised
`AttributeError` on the batched entry point while the single-state path worked. Beyond the missing
method, one session run per state is the slow way to score many states: ONNX Runtime already
parallelises inside a run, so the rows of several states belong in one call.

No onnxruntime and no checkpoint: onnxruntime is imported lazily inside `__init__`, so the class
imports, and `predict_batch` is driven with a stub session and a fake tokenizer (the technique
`tests/test_onnx_lang_parity.py` uses for `_infer`). The stub derives each row's logits from that
row's own tokens, so a row decoded for the wrong state changes the answer and fails these checks.

Run: python tests/test_onnx_batch.py
"""
import inspect
import os
import sys
import threading

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import numpy as np  # noqa: E402

from laya.agent import Agent  # noqa: E402
from laya.common import clamp_temperature  # noqa: E402
from laya.onnx_agent import ONNXAgent  # noqa: E402
from laya.router import Router  # noqa: E402

PASS, FAIL = [], []


def check(name, got, want):
    if got == want:
        PASS.append(name)
    else:
        FAIL.append("%s:\n     got  %r\n     want %r" % (name, got, want))


def check_true(name, cond, detail=""):
    if cond:
        PASS.append(name)
    else:
        FAIL.append("%s %s" % (name, detail))


def check_raises(name, exc, fn):
    try:
        fn()
    except exc:
        PASS.append(name)
    except Exception as e:  # noqa: BLE001
        FAIL.append("%s: raised %r, expected %s" % (name, e, exc.__name__))
    else:
        FAIL.append("%s: did not raise %s" % (name, exc.__name__))


# ---------------------------------------------------------------- a fake tokenizer + stub session
class _FakeTok:
    cls_token_id, sep_token_id, mask_token_id, pad_token_id = 1, 2, 3, 0
    mask_token = "[M]"

    def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):
        ids = [10 + (ord(c) % 40) for c in text]
        if truncation and max_length:
            ids = ids[:max_length]
        return {"input_ids": ids}


class _StubSession:
    """Row logits derived from the row's own tokens: a mis-mapped row changes the answer."""

    def __init__(self):
        self.calls = []          # rows per session run, in order

    def run(self, names, inputs):
        ids = inputs["input_ids"]
        self.calls.append(len(ids))
        # Every row for a state carries that state's tokens last, so this weight is constant
        # within a state and distinct across the states used below.
        weight = 1.0 + (ids.sum(axis=1) % 4)
        logits = np.stack([weight, np.ones_like(weight)], axis=1).astype(np.float32)
        act = np.tile(np.array([[0.25, 0.75]], dtype=np.float32), (len(ids), 1))
        return [logits, act]


def _bare_onnx(lang_temperatures=None):
    a = ONNXAgent.__new__(ONNXAgent)         # skip __init__: no onnxruntime, no checkpoint
    a.model_id = "stub"
    a.cfg = {"max_len": 64, "head_max_len": 32}
    a.tok = _FakeTok()
    a.temperature = [1.0, 1.0, 1.0]
    a.temperature_by_options = {}
    a.session = _StubSession()
    a.hooks = ()
    a.hooks_raise = True
    a.hooks_concurrent = True
    a.hooks_timeout = None
    a._hooks_lock = None
    a._hooks_mutex = threading.Lock()
    a.lang_temperatures = {}
    for l, cfg in (lang_temperatures or {}).items():
        norm = l.split("-")[0].lower()
        a.lang_temperatures[norm] = {
            "temperature": [clamp_temperature(t) for t in cfg["temperature"]],
            "temperature_by_options": {k: clamp_temperature(v)
                                       for k, v in cfg.get("temperature_by_options", {}).items()},
        }
    return a


QUESTIONS = {
    "dept": {"type": "choice", "instructions": "Which team?",
             "criteria": {"billing": "money", "support": "help", "sales": "buy"}},
    "flag": {"type": "noul", "instructions": "Urgent?"},
}
STATES = ["aa", "bb", "c", "e"]   # state token sums 54/56/29/31 -> four distinct mod-4 weights

# ---------------------------------------------------------------- contract
check_true("contract/predict_batch is defined", callable(getattr(ONNXAgent, "predict_batch", None)))
check_true("contract/system_one delegates to predict_batch",
           "self.predict_batch(" in inspect.getsource(ONNXAgent.system_one))
check_true("contract/predict alias preserved", ONNXAgent.predict is ONNXAgent.system_one)

_onnx_so = inspect.signature(ONNXAgent.predict_batch).parameters
_agent_so = inspect.signature(Agent.predict_batch).parameters
_shared = ["batch_size", "lang", "hooks", "on_predict_start", "on_predict_end", "hooks_raise",
           "max_len", "head_max_len", "min_confidence"]
check("contract/parameter names match Agent.predict_batch",
      [p for p in _shared if p not in _onnx_so], [])
check("contract/the Agent's full knob set is now ported",
      sorted(set(_agent_so) - set(_onnx_so)), [])


# ---------------------------------------------------------------- batched == sequential
sequential_agent = _bare_onnx()
sequential = [sequential_agent.system_one(s, QUESTIONS) for s in STATES]

agent = _bare_onnx()
batched = agent.predict_batch(STATES, QUESTIONS)

check("batch/one result per state", len(batched), len(STATES))
check("batch/answers match system_one exactly", batched, sequential)
check("batch/four states cost one session run", agent.session.calls, [2 * len(STATES)])
check_true("batch/result shape matches system_one",
           all(set(r) == {"model", "answers", "usage"} for r in batched))
check("batch/usage matches system_one", [r["usage"] for r in batched],
      [r["usage"] for r in sequential])
check_true("batch/states are distinguishable (so the parity check is not vacuous)",
           len({r["answers"]["dept"]["probabilities"]["billing"] for r in batched}) == len(STATES))

chunked = _bare_onnx()
chunked_results = chunked.predict_batch(STATES, QUESTIONS, batch_size=2)
check("batch/batch_size=2 chunks into two runs of two states each",
      chunked.session.calls, [2 * 2, 2 * 2])
check("batch/chunked results keep input order", chunked_results, sequential)
for batch_size in (0, -1, None):
    whole = _bare_onnx()
    check("batch/batch_size=%r still matches system_one" % batch_size,
          whole.predict_batch(STATES, QUESTIONS, batch_size=batch_size), sequential)
    check("batch/batch_size=%r does not chunk" % batch_size, whole.session.calls, [2 * len(STATES)])

# ---------------------------------------------------------------- empty and invalid input
empty = _bare_onnx()
check("empty/no states -> empty list", empty.predict_batch([], QUESTIONS), [])
check("empty/no states -> no session run", empty.session.calls, [])
no_q = _bare_onnx()
check("empty/no questions -> one zero-usage result per state",
      no_q.predict_batch(STATES, {}),
      [{"model": "laya-rl-agent-onnx", "answers": {},
        "usage": {"input_tokens": 0, "output_tokens": 0}}] * len(STATES))
check("empty/no questions -> no session run", no_q.session.calls, [])
check_raises("empty/bare string rejected", TypeError,
             lambda: _bare_onnx().predict_batch("just a string", QUESTIONS))
check_raises("empty/bare dict rejected", TypeError,
             lambda: _bare_onnx().predict_batch({"body": "x"}, QUESTIONS))
check_raises("empty/invalid question rejected before any run", ValueError,
             lambda: _bare_onnx().predict_batch(STATES, {"bad": {"type": "nope", "instructions": "?"}}))
for bad_type in ([], {}):
    invalid = _bare_onnx()
    name = "invalid/question type %r" % (bad_type,)
    try:
        invalid.predict_batch(STATES, {"refund": {"type": bad_type, "instructions": "Refund requested?"}})
        FAIL.append("%s: no error raised" % name)
    except ValueError as e:
        check(name, str(e), "question 'refund': unknown type %r; use one of ['choice', 'noul', 'score']"
              % (bad_type,))
    except Exception as e:
        FAIL.append("%s: %s instead of ValueError: %s" % (name, type(e).__name__, e))
    check("%s does not run the session" % name, invalid.session.calls, [])
for bad_ins in (None, "", "   ", []):
    check_raises("invalid/instructions %r rejected" % (bad_ins,), ValueError,
                 lambda bi=bad_ins: _bare_onnx().predict_batch(STATES, {"q": {"type": "noul", "instructions": bi}}))
check_raises("invalid/questions not a dict", TypeError,
             lambda: _bare_onnx().predict_batch(STATES, ["not", "a", "dict"]))
check_raises("invalid/state is None in batch", TypeError,
             lambda: _bare_onnx().predict_batch([None], QUESTIONS))
check("budget/max_len and head_max_len forward through the batch path",
      _bare_onnx().predict_batch(STATES, QUESTIONS, max_len=32, head_max_len=16),
      [_bare_onnx().system_one(s, QUESTIONS, max_len=32, head_max_len=16) for s in STATES])

# ---------------------------------------------------------------- hooks
seen, events = [], []


def _end_hook(ctx):
    events.append(ctx)
    seen.append([sorted(r["answers"]) for r in ctx.results])


hooked = _bare_onnx()
hooked_results = hooked.predict_batch(STATES[:2], QUESTIONS, on_predict_end=_end_hook)
check("hooks/end hook sees the batched results", seen[-1], [["dept", "flag"]] * 2)
check("hooks/end hook usage aggregates every state", events[-1].usage["input_tokens"],
      sum(r["usage"]["input_tokens"] for r in hooked_results))
check_true("hooks/end hook sees elapsed_ms", isinstance(events[-1].elapsed_ms, float))

rewritten = _bare_onnx()
check("hooks/start hook can rewrite the states",
      rewritten.predict_batch(STATES[:1], QUESTIONS,
                              on_predict_start=lambda ctx: setattr(ctx, "states", ["fff"])),
      [_bare_onnx().system_one("fff", QUESTIONS)])

skipped = _bare_onnx()
sentinel = [{"model": "cached", "answers": {}, "usage": {"input_tokens": 0, "output_tokens": 0}}]
check("hooks/start hook can serve cached results",
      skipped.predict_batch(STATES, QUESTIONS, on_predict_start=lambda ctx: ctx.skip(sentinel)),
      sentinel)
check("hooks/skip runs no session", skipped.session.calls, [])


class _ErrorHook:
    def __init__(self, sink):
        self.sink = sink

    def on_error(self, ctx):
        self.sink.append(ctx.error)


class _Boom:
    def run(self, names, inputs):
        raise RuntimeError("session failed")


errors = []
failing = _bare_onnx()
failing.session = _Boom()
check_raises("hooks/session failure propagates", RuntimeError,
             lambda: failing.predict_batch(STATES, QUESTIONS, hooks=[_ErrorHook(errors)]))
check("hooks/error hook receives the failure once", [str(e) for e in errors], ["session failed"])


# ---------------------------------------------------------------- min_confidence
gated = _bare_onnx()
results = gated.predict_batch(STATES, QUESTIONS, min_confidence=0.9)
flagged = [r["answers"]["dept"].get("low_confidence", False) for r in results]
check("min_confidence/flags exactly the answers below the threshold", flagged,
      [r["answers"]["dept"]["answer_confidence"] < 0.9 for r in results])
check_true("min_confidence/covers both branches", any(flagged) and not all(flagged), flagged)
check("min_confidence/raw answers stay intact",
      [r["answers"]["dept"]["probabilities"] for r in results],
      [r["answers"]["dept"]["probabilities"] for r in sequential])

flag_seen = []
_bare_onnx().predict_batch(
    STATES, QUESTIONS, min_confidence=0.9,
    on_predict_end=lambda ctx: flag_seen.append(
        [r["answers"]["dept"].get("low_confidence", False) for r in ctx.results]))
check("min_confidence/end hook sees the flags", flag_seen[-1], flagged)

check("min_confidence/unset adds no flag",
      any("low_confidence" in a for r in _bare_onnx().predict_batch(STATES, QUESTIONS)
          for a in r["answers"].values()), False)
check("min_confidence/0.0 is a no-op",
      any("low_confidence" in a
          for r in _bare_onnx().predict_batch(STATES, QUESTIONS, min_confidence=0.0)
          for a in r["answers"].values()), False)
check_raises("min_confidence/rejects out-of-range", ValueError,
             lambda: _bare_onnx().predict_batch(STATES, QUESTIONS, min_confidence=1.5))
check_raises("min_confidence/rejects a bool", ValueError,
             lambda: _bare_onnx().predict_batch(STATES, QUESTIONS, min_confidence=True))

# ---------------------------------------------------------------- lang on the batch path
lang_agent = _bare_onnx({"de": {"temperature": [3.0, 3.0, 3.0]}})
base_lang = lang_agent.predict_batch(STATES, QUESTIONS)
de_lang = lang_agent.predict_batch(STATES, QUESTIONS, lang="de")
check("lang/batch equals system_one with the same lang",
      de_lang, [lang_agent.system_one(s, QUESTIONS, lang="de") for s in STATES])
check_true("lang/override changes the batch probabilities",
           [r["answers"]["dept"]["probabilities"] for r in de_lang]
           != [r["answers"]["dept"]["probabilities"] for r in base_lang])
check("lang/hyphen subtag resolves on the batch path",
      lang_agent.predict_batch(STATES, QUESTIONS, lang="de-DE")[0]["answers"]["dept"]["probabilities"],
      de_lang[0]["answers"]["dept"]["probabilities"])


# ---------------------------------------------------------------- Router integration
# The motivating failure: Router.predict_batch calls agent.predict_batch unconditionally, so an
# attached ONNX agent raised AttributeError on the batched entry point while predict worked.
router_agent = _bare_onnx()
router = Router(max_loaded=1, default="english")
router.attach("english", router_agent)
requests = [{"state": s, "questions": QUESTIONS, "model": "english"} for s in STATES[:2]]
routed = router.predict_batch(requests)

check("router/one routed result per request", len(routed), 2)
check("router/results carry the routing key", [sorted(r) for r in routed],
      [["answers", "model", "routing", "usage"]] * 2)
check("router/two states share one session run", router_agent.session.calls, [2 * 2])


# ---------------------------------------------------------------- single-option questions
# The exported graph traces the decision head's `topk(2)` (the eager forward's one-option branch is
# a Python `if` on the marker count, which the trace bakes in), so ONNX Runtime refused a run whose
# questions all had one option -- "k argument [2] should not be greater than specified axis dim
# value [1]" -- while the PyTorch Agent answered it. This stub enforces the same rule, and scores a
# masked slot -1e4 the way the graph's `masked_fill` does.
class _ExportedGraphSession:
    def __init__(self):
        self.marker_widths = []
        self.masks = []

    def run(self, names, inputs):
        mask = inputs["marker_mask"]
        self.marker_widths.append(mask.shape[1])
        self.masks.append(mask.tolist())
        if mask.shape[1] < 2:
            raise RuntimeError("[ONNXRuntimeError] : 1 : FAIL : Non-zero status code returned while "
                               "running TopK node. Status Message: k argument [2] should not be "
                               "greater than specified axis dim value [%d]" % mask.shape[1])
        logits = np.where(mask, 1.0 + np.arange(mask.shape[1], dtype=np.float32), -1e4).astype(np.float32)
        act = np.tile(np.array([[0.25, 0.75]], dtype=np.float32), (len(mask), 1))
        return [logits, act]


ONE_OPTION = {"dept": {"type": "choice", "instructions": "Which team?", "criteria": {"billing": "money"}}}
ONE_LEVEL = {"level": {"type": "score", "instructions": "How urgent?", "criteria": ["not urgent"]}}

for label, questions, key, want in (("choice", ONE_OPTION, "dept", {"billing": 1.0}),
                                    ("score", ONE_LEVEL, "level", {"0": 1.0})):
    single = _bare_onnx()
    single.session = _ExportedGraphSession()
    try:
        got = single.predict("aa", questions)["answers"][key]["probabilities"]
    except RuntimeError as e:
        got = str(e)[:80]
    check("single option/a one-option %s is answered, not refused by the graph" % label, got, want)
    check("single option/%s: the extra slot is padded and masked" % label, single.session.masks, [[[True, False]]])

single = _bare_onnx()
single.session = _ExportedGraphSession()
try:
    got = [r["answers"]["dept"]["probabilities"] for r in single.predict_batch(STATES, ONE_OPTION)]
except RuntimeError as e:
    got = str(e)[:80]
check("single option/the batch path answers too, in one run", (got, single.session.marker_widths),
      ([{"billing": 1.0}] * len(STATES), [2]))

wide = _bare_onnx()
wide.session = _ExportedGraphSession()
wide.predict("aa", QUESTIONS)
check("single option/a run that already has two or more slots is sent as before",
      wide.session.marker_widths, [3])


# ---------------------------------------------------------------- question token reuse
# Agent.predict_batch caches question tokenizations across states (@_reuse_question_tokens);
# ONNXAgent.predict_batch shares that cache, so question instructions and options are encoded once
# per batch rather than once per state document.
tok_records = []


class _TrackingTok(_FakeTok):
    def __call__(self, text, **kw):
        tok_records.append(text)
        return super().__call__(text, **kw)


tracking_agent = _bare_onnx()
tracking_agent.tok = _TrackingTok()
tracking_agent.predict_batch(STATES, QUESTIONS)
dept_instruction = "choice question: Which team?"
check("reuse/question instruction encoded once across the batch",
      tok_records.count(dept_instruction), 1)
check("reuse/total tokenizations capped by reuse across states",
      len(tok_records), 7 + len(STATES))



# --------------------------------------------------------------------------- report
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
    print("  FAIL " + f)
sys.exit(1 if FAIL else 0)