Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(evals): score an ONNX export with laya-evals run --onnx

Aashisha87341396901
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7a9cca5f..cb53fb38 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -183,6 +183,12 @@ jobs:
- name: Eval API guard
run: python tests/test_evals_api.py
+ # The ONNX runner suite fakes the agent, so it stays weight-free and needs no
+ # onnxruntime: the module only imports numpy, and ONNXAgent's constructor is
+ # never called.
+ - name: Eval ONNX runner tests
+ run: python tests/test_evals_onnx.py
+
- name: Fixture dataset validates
run: laya-evals validate research/evals/fixture.jsonl
diff --git a/docs/evals.md b/docs/evals.md
index 3660ec3a..6c1e2496 100644
--- a/docs/evals.md
+++ b/docs/evals.md
@@ -29,6 +29,36 @@ Exit codes: `0` on success, `1` when a threshold or a baseline tolerance fails,
usage error. `run` prints the overall metrics and any requested slices to stdout, and writes
the full report and a Markdown summary when `--json` / `--markdown` are given.
+## Evaluating an ONNX export
+
+`run --onnx PATH` scores an exported ONNX model through `ONNXAgent` instead of the torch
+Router, so an ONNX deployment (including an INT8 copy from `scripts/export_onnx.py --quantize`)
+gets gated by the same thresholds and baselines as the torch path:
+
+```bash
+python scripts/export_onnx.py --model convaiinnovations/laya --output laya.onnx --quantize
+laya-evals run data.jsonl --onnx laya.int8.onnx --max-ece 0.05
+```
+
+`--model` names the checkpoint the export came from — a Hub id or local path, not a Router
+short name like `english`, since there is no Router on this path (default
+`convaiinnovations/laya`). Its config and tokenizer are loaded from there. The agent serves one checkpoint, so a dataset row
+whose `model` field names a different one fails with a clear error rather than being silently
+answered by the wrong model; `--device` does not apply. `--batch-size` uses the agent's batch
+API when it has one and falls back to one call per state otherwise. The report's `config` block
+records the `onnx` path.
+
+Measured on `research/evals/fixture.jsonl` (12 labelled rows, English checkpoint, CPU):
+
+| runner | choice_acc | noul_acc | score_mae | ece | mean_conf | p50 ms |
+|---|---|---|---|---|---|---|
+| torch Router | 0.75 | 1.00 | 1.3418 | 0.1596 | 0.7304 | 116.8 |
+| `--onnx` fp32 | 0.75 | 1.00 | 1.3418 | 0.1596 | 0.7304 | 66.3 |
+| `--onnx` int8 | 0.75 | 1.00 | 1.3512 | 0.1658 | 0.7304 | 46.3 |
+
+The fp32 export reproduces the torch numbers exactly, and the quantized copy moves `score_mae`
+by 0.009 and `ece` by 0.006 — the kind of drift `compare --tolerance` is meant to gate.
+
## Dataset format
One JSON object per line (JSONL). Blank lines and lines starting with `#` are ignored.
diff --git a/laya/evals_cli.py b/laya/evals_cli.py
index e656eba6..396ac384 100644
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@ -34,6 +34,37 @@ class RouterRunner:
return self.router.predict_batch(requests, batch_size=batch_size)
+class OnnxRunner:
+ """Adapt a single-checkpoint `ONNXAgent` to the harness, like `RouterRunner` does for a Router.
+
+ The agent serves one checkpoint, so a per-example `model` that names a different one is an
+ error rather than a silent no-op: the report would otherwise claim to score a fleet it never
+ ran. `predict_batch` delegates to the agent when it has one and falls back to one predict per
+ state otherwise, so the harness's batching path works on every ONNXAgent build.
+ """
+
+ def __init__(self, agent: Any):
+ self.agent = agent
+
+ def _check_model(self, model: Optional[str]) -> None:
+ if model is not None and model != self.agent.model_id:
+ raise EvalError(
+ "the ONNX runner serves only %r, but this example asks for %r; "
+ "run them separately or drop --onnx" % (self.agent.model_id, model))
+
+ def predict(self, state: Any, questions: Dict[str, Any], model: Optional[str] = None) -> Dict[str, Any]:
+ self._check_model(model)
+ return self.agent.predict(state, questions)
+
+ def predict_batch(self, states: Sequence[Any], questions: Dict[str, Any],
+ model: Optional[str] = None, batch_size: Optional[int] = None) -> List[Dict[str, Any]]:
+ self._check_model(model)
+ agent_batch = getattr(self.agent, "predict_batch", None)
+ if agent_batch is not None:
+ return agent_batch(list(states), questions, batch_size=batch_size)
+ return [self.agent.predict(state, questions) for state in states]
+
+
def _parse_pairs(pairs: Optional[Sequence[str]]) -> Dict[str, float]:
out: Dict[str, float] = {}
for pair in pairs or []:
@@ -59,6 +90,10 @@ def _build_parser() -> argparse.ArgumentParser:
run.add_argument("dataset")
run.add_argument("--model", help="force a checkpoint instead of auto-routing")
run.add_argument("--device", help="torch device, e.g. cpu or cuda")
+ run.add_argument("--onnx", metavar="PATH",
+ help="evaluate an ONNX export through ONNXAgent instead of the torch Router; "
+ "--model then names the checkpoint directory or Hub id the export came "
+ "from (default convaiinnovations/laya)")
run.add_argument("--batch-size", type=int, help="examples per forward pass when questions match")
run.add_argument("--on-error", choices=("fail", "skip"), default="fail")
run.add_argument("--baseline", help="a baseline report JSON to compare against")
@@ -116,14 +151,22 @@ def _cmd_validate(args) -> int:
def _cmd_run(args) -> int:
- import laya
-
dataset = evals.Dataset.from_jsonl(args.dataset)
if args.model:
for example in dataset.examples: # --model is authoritative over per-row model
example.model = args.model
- runner: Any = RouterRunner(laya.Router(device=args.device, preload=False))
+ if args.onnx:
+ from .onnx_agent import ONNXAgent
+ # The export was produced from one checkpoint; --model names it (the ONNXAgent load
+ # needs its config and tokenizer), so the default is the english bundle repo.
+ agent = ONNXAgent(args.model or "convaiinnovations/laya", onnx_path=args.onnx)
+ runner: Any = OnnxRunner(agent)
+ else:
+ import laya
+ runner = RouterRunner(laya.Router(device=args.device, preload=False))
config = {"dataset": args.dataset, "model": args.model, "device": args.device}
+ if args.onnx:
+ config["onnx"] = args.onnx
report = evals.evaluate(runner, dataset, batch_size=args.batch_size,
on_error=args.on_error, config=config)
diff --git a/tests/test_evals_onnx.py b/tests/test_evals_onnx.py
new file mode 100644
index 00000000..d3b25684
--- /dev/null
+++ b/tests/test_evals_onnx.py
@@ -0,0 +1,161 @@
+"""`laya-evals run --onnx`: the eval harness can score an ONNX export, not just the Router.
+
+`OnnxRunner` adapts a single-checkpoint `ONNXAgent` to the harness contract (predict /
+predict_batch with an optional per-example model). Weight-free throughout: the agent is a fake,
+and the CLI end-to-end tests monkeypatch `laya.onnx_agent.ONNXAgent` -- importing that module
+needs only numpy, onnxruntime is loaded inside its constructor -- so the real wiring from
+`main(["run", ..., "--onnx", ...])` down to the written report JSON is exercised without
+onnxruntime or a checkpoint download.
+
+Run: python tests/test_evals_onnx.py
+"""
+import json
+import os
+import sys
+import tempfile
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from laya import evals_cli # noqa: E402
+from laya.evals import EvalError # noqa: E402
+from laya.evals_cli import OnnxRunner # noqa: E402
+
+PASS, FAIL = [], []
+
+
+def check(name, got, want):
+ if got == want:
+ PASS.append(name)
+ else:
+ FAIL.append("%s:\n got %r\n want %r" % (name, got, want))
+
+
+def check_true(name, cond, detail=""):
+ if cond:
+ PASS.append(name)
+ else:
+ FAIL.append("%s %s" % (name, detail))
+
+
+class _FakeAgent:
+ """Duck-typed ONNXAgent: records calls, answers every choice question with `label`."""
+
+ def __init__(self, model_id_or_path, onnx_path=None, **kw):
+ self.model_id = model_id_or_path
+ self.onnx_path = onnx_path
+ self.calls = []
+
+ def _answer(self, questions):
+ return {"model": "fake-onnx",
+ "answers": {qid: {"type": "choice", "choice": "billing", "answer_confidence": 0.9}
+ for qid in questions},
+ "usage": {"input_tokens": 3, "output_tokens": 0}}
+
+ def predict(self, state, questions, **kw):
+ self.calls.append(("predict", state))
+ return self._answer(questions)
+
+
+class _BatchAgent(_FakeAgent):
+ def predict_batch(self, states, questions, batch_size=None):
+ self.calls.append(("predict_batch", tuple(states), batch_size))
+ return [self._answer(questions) for _ in states]
+
+
+# ---------------------------------------------------------------- predict contract
+agent = _FakeAgent("english-ckpt")
+runner = OnnxRunner(agent)
+Q = {"intent": {"type": "choice", "instructions": "i", "criteria": {"billing": "b"}}}
+res = runner.predict("a state", Q)
+check("predict/returns the agent answer", res["answers"]["intent"]["choice"], "billing")
+check("predict/forwards state to the agent", agent.calls[-1], ("predict", "a state"))
+runner.predict("s2", Q, model="english-ckpt")
+check_true("predict/matching per-example model is accepted", True)
+
+try:
+ runner.predict("s3", Q, model="multilingual")
+ check_true("predict/foreign model raises EvalError", False, "no exception")
+except EvalError as exc:
+ msg = str(exc)
+ check_true("predict/foreign model raises EvalError", True)
+ check_true("predict/error names the served checkpoint", "english-ckpt" in msg, msg)
+ check_true("predict/error names the asked-for checkpoint", "multilingual" in msg, msg)
+
+# ---------------------------------------------------------------- predict_batch contract
+agent_nobatch = _FakeAgent("ckpt")
+OnnxRunner(agent_nobatch).predict_batch(["a", "b", "c"], Q)
+check("predict_batch/falls back to one predict per state",
+ [c[1] for c in agent_nobatch.calls], ["a", "b", "c"])
+
+bagent = _BatchAgent("ckpt")
+out = OnnxRunner(bagent).predict_batch(["a", "b"], Q, batch_size=2)
+check("predict_batch/delegates once when the agent can batch",
+ bagent.calls, [("predict_batch", ("a", "b"), 2)])
+check("predict_batch/delegated result passes through", len(out), 2)
+try:
+ OnnxRunner(bagent).predict_batch(["a"], Q, model="other")
+ check_true("predict_batch/foreign model raises too", False, "no exception")
+except EvalError:
+ check_true("predict_batch/foreign model raises too", True)
+
+# ---------------------------------------------------------------- CLI wiring
+args = evals_cli._build_parser().parse_args(["run", "d.jsonl", "--onnx", "m.onnx"])
+check("cli/--onnx parses on the run subcommand", args.onnx, "m.onnx")
+args = evals_cli._build_parser().parse_args(["run", "d.jsonl"])
+check("cli/--onnx defaults to None (torch Router path unchanged)", args.onnx, None)
+
+tmp = tempfile.mkdtemp(prefix="laya_evals_onnx_")
+dataset = os.path.join(tmp, "data.jsonl")
+with open(dataset, "w") as f:
+ f.write(json.dumps({"state": "charged twice, refund me",
+ "questions": Q, "expected": {"intent": "billing"}}) + "\n")
+ f.write(json.dumps({"state": "billed again after cancelling",
+ "questions": Q, "expected": {"intent": "billing"}}) + "\n")
+report_path = os.path.join(tmp, "report.json")
+
+import laya.onnx_agent as onnx_agent_module # noqa: E402 (numpy-only at import time)
+
+_real_onnx_agent = onnx_agent_module.ONNXAgent
+try:
+ onnx_agent_module.ONNXAgent = _FakeAgent
+ rc = evals_cli.main(["run", dataset, "--onnx", os.path.join(tmp, "laya.onnx"),
+ "--model", "english-ckpt", "--json", report_path])
+ check("cli/run --onnx exits 0 on a passing dataset", rc, 0)
+ with open(report_path) as f:
+ report = json.load(f)
+ check("cli/report records the onnx export path",
+ report["config"]["onnx"], os.path.join(tmp, "laya.onnx"))
+ check("cli/report keeps the forced checkpoint", report["config"]["model"], "english-ckpt")
+ check("cli/metrics are computed from the ONNX answers",
+ report["overall"]["choice_accuracy"], 1.0)
+
+ # --model omitted: the export's checkpoint defaults to the english bundle repo.
+ rc = evals_cli.main(["run", dataset, "--onnx", "laya.onnx"])
+ check("cli/run without --model still scores", rc, 0)
+
+ # batching reaches the runner's predict_batch through the harness
+ onnx_agent_module.ONNXAgent = _BatchAgent
+ rc = evals_cli.main(["run", dataset, "--onnx", "laya.onnx", "--model", "ckpt",
+ "--batch-size", "2"])
+ check("cli/--batch-size runs the ONNX batch path", rc, 0)
+
+ # a per-example model the single-checkpoint agent does not serve is an error, not a no-op
+ # (no --model here: --model would authoritatively rewrite every row and mask the mismatch)
+ mismatch = os.path.join(tmp, "mismatch.jsonl")
+ with open(mismatch, "w") as f:
+ row = json.loads(open(dataset).readline())
+ row["model"] = "some-other-checkpoint"
+ f.write(json.dumps(row) + "\n")
+ rc = evals_cli.main(["run", mismatch, "--onnx", "laya.onnx"])
+ check("cli/foreign per-example model exits 1", rc, 1)
+finally:
+ onnx_agent_module.ONNXAgent = _real_onnx_agent
+
+# ---------------------------------------------------------------- torch path unchanged
+check("api/RouterRunner still exists beside OnnxRunner", hasattr(evals_cli, "RouterRunner"), True)
+
+# ---------------------------------------------------------------- report
+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
+for f in FAIL:
+ print(" FAIL " + f)
+sys.exit(1 if FAIL else 0)