Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(evals): score an ONNX export with laya-evals run --onnx
Aashisha87341396901
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 7a9cca5f..cb53fb38 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -183,6 +183,12 @@ jobs:- name: Eval API guardrun: python tests/test_evals_api.py+ # The ONNX runner suite fakes the agent, so it stays weight-free and needs no+ # onnxruntime: the module only imports numpy, and ONNXAgent's constructor is+ # never called.+ - name: Eval ONNX runner tests+ run: python tests/test_evals_onnx.py+- name: Fixture dataset validatesrun: laya-evals validate research/evals/fixture.jsonldiff --git a/docs/evals.md b/docs/evals.mdindex 3660ec3a..6c1e2496 100644--- a/docs/evals.md+++ b/docs/evals.md@@ -29,6 +29,36 @@ Exit codes: `0` on success, `1` when a threshold or a baseline tolerance fails,usage error. `run` prints the overall metrics and any requested slices to stdout, and writesthe full report and a Markdown summary when `--json` / `--markdown` are given.+## Evaluating an ONNX export++`run --onnx PATH` scores an exported ONNX model through `ONNXAgent` instead of the torch+Router, so an ONNX deployment (including an INT8 copy from `scripts/export_onnx.py --quantize`)+gets gated by the same thresholds and baselines as the torch path:++```bash+python scripts/export_onnx.py --model convaiinnovations/laya --output laya.onnx --quantize+laya-evals run data.jsonl --onnx laya.int8.onnx --max-ece 0.05+```++`--model` names the checkpoint the export came from — a Hub id or local path, not a Router+short name like `english`, since there is no Router on this path (default+`convaiinnovations/laya`). Its config and tokenizer are loaded from there. The agent serves one checkpoint, so a dataset row+whose `model` field names a different one fails with a clear error rather than being silently+answered by the wrong model; `--device` does not apply. `--batch-size` uses the agent's batch+API when it has one and falls back to one call per state otherwise. The report's `config` block+records the `onnx` path.++Measured on `research/evals/fixture.jsonl` (12 labelled rows, English checkpoint, CPU):++| runner | choice_acc | noul_acc | score_mae | ece | mean_conf | p50 ms |+|---|---|---|---|---|---|---|+| torch Router | 0.75 | 1.00 | 1.3418 | 0.1596 | 0.7304 | 116.8 |+| `--onnx` fp32 | 0.75 | 1.00 | 1.3418 | 0.1596 | 0.7304 | 66.3 |+| `--onnx` int8 | 0.75 | 1.00 | 1.3512 | 0.1658 | 0.7304 | 46.3 |++The fp32 export reproduces the torch numbers exactly, and the quantized copy moves `score_mae`+by 0.009 and `ece` by 0.006 — the kind of drift `compare --tolerance` is meant to gate.+## Dataset formatOne JSON object per line (JSONL). Blank lines and lines starting with `#` are ignored.diff --git a/laya/evals_cli.py b/laya/evals_cli.pyindex e656eba6..396ac384 100644--- a/laya/evals_cli.py+++ b/laya/evals_cli.py@@ -34,6 +34,37 @@ class RouterRunner:return self.router.predict_batch(requests, batch_size=batch_size)+class OnnxRunner:+ """Adapt a single-checkpoint `ONNXAgent` to the harness, like `RouterRunner` does for a Router.++ The agent serves one checkpoint, so a per-example `model` that names a different one is an+ error rather than a silent no-op: the report would otherwise claim to score a fleet it never+ ran. `predict_batch` delegates to the agent when it has one and falls back to one predict per+ state otherwise, so the harness's batching path works on every ONNXAgent build.+ """++ def __init__(self, agent: Any):+ self.agent = agent++ def _check_model(self, model: Optional[str]) -> None:+ if model is not None and model != self.agent.model_id:+ raise EvalError(+ "the ONNX runner serves only %r, but this example asks for %r; "+ "run them separately or drop --onnx" % (self.agent.model_id, model))++ def predict(self, state: Any, questions: Dict[str, Any], model: Optional[str] = None) -> Dict[str, Any]:+ self._check_model(model)+ return self.agent.predict(state, questions)++ def predict_batch(self, states: Sequence[Any], questions: Dict[str, Any],+ model: Optional[str] = None, batch_size: Optional[int] = None) -> List[Dict[str, Any]]:+ self._check_model(model)+ agent_batch = getattr(self.agent, "predict_batch", None)+ if agent_batch is not None:+ return agent_batch(list(states), questions, batch_size=batch_size)+ return [self.agent.predict(state, questions) for state in states]++def _parse_pairs(pairs: Optional[Sequence[str]]) -> Dict[str, float]:out: Dict[str, float] = {}for pair in pairs or []:@@ -59,6 +90,10 @@ def _build_parser() -> argparse.ArgumentParser:run.add_argument("dataset")run.add_argument("--model", help="force a checkpoint instead of auto-routing")run.add_argument("--device", help="torch device, e.g. cpu or cuda")+ run.add_argument("--onnx", metavar="PATH",+ help="evaluate an ONNX export through ONNXAgent instead of the torch Router; "+ "--model then names the checkpoint directory or Hub id the export came "+ "from (default convaiinnovations/laya)")run.add_argument("--batch-size", type=int, help="examples per forward pass when questions match")run.add_argument("--on-error", choices=("fail", "skip"), default="fail")run.add_argument("--baseline", help="a baseline report JSON to compare against")@@ -116,14 +151,22 @@ def _cmd_validate(args) -> int:def _cmd_run(args) -> int:- import laya-dataset = evals.Dataset.from_jsonl(args.dataset)if args.model:for example in dataset.examples: # --model is authoritative over per-row modelexample.model = args.model- runner: Any = RouterRunner(laya.Router(device=args.device, preload=False))+ if args.onnx:+ from .onnx_agent import ONNXAgent+ # The export was produced from one checkpoint; --model names it (the ONNXAgent load+ # needs its config and tokenizer), so the default is the english bundle repo.+ agent = ONNXAgent(args.model or "convaiinnovations/laya", onnx_path=args.onnx)+ runner: Any = OnnxRunner(agent)+ else:+ import laya+ runner = RouterRunner(laya.Router(device=args.device, preload=False))config = {"dataset": args.dataset, "model": args.model, "device": args.device}+ if args.onnx:+ config["onnx"] = args.onnxreport = evals.evaluate(runner, dataset, batch_size=args.batch_size,on_error=args.on_error, config=config)diff --git a/tests/test_evals_onnx.py b/tests/test_evals_onnx.pynew file mode 100644index 00000000..d3b25684--- /dev/null+++ b/tests/test_evals_onnx.py@@ -0,0 +1,161 @@+"""`laya-evals run --onnx`: the eval harness can score an ONNX export, not just the Router.++`OnnxRunner` adapts a single-checkpoint `ONNXAgent` to the harness contract (predict /+predict_batch with an optional per-example model). Weight-free throughout: the agent is a fake,+and the CLI end-to-end tests monkeypatch `laya.onnx_agent.ONNXAgent` -- importing that module+needs only numpy, onnxruntime is loaded inside its constructor -- so the real wiring from+`main(["run", ..., "--onnx", ...])` down to the written report JSON is exercised without+onnxruntime or a checkpoint download.++Run: python tests/test_evals_onnx.py+"""+import json+import os+import sys+import tempfile++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from laya import evals_cli # noqa: E402+from laya.evals import EvalError # noqa: E402+from laya.evals_cli import OnnxRunner # noqa: E402++PASS, FAIL = [], []+++def check(name, got, want):+ if got == want:+ PASS.append(name)+ else:+ FAIL.append("%s:\n got %r\n want %r" % (name, got, want))+++def check_true(name, cond, detail=""):+ if cond:+ PASS.append(name)+ else:+ FAIL.append("%s %s" % (name, detail))+++class _FakeAgent:+ """Duck-typed ONNXAgent: records calls, answers every choice question with `label`."""++ def __init__(self, model_id_or_path, onnx_path=None, **kw):+ self.model_id = model_id_or_path+ self.onnx_path = onnx_path+ self.calls = []++ def _answer(self, questions):+ return {"model": "fake-onnx",+ "answers": {qid: {"type": "choice", "choice": "billing", "answer_confidence": 0.9}+ for qid in questions},+ "usage": {"input_tokens": 3, "output_tokens": 0}}++ def predict(self, state, questions, **kw):+ self.calls.append(("predict", state))+ return self._answer(questions)+++class _BatchAgent(_FakeAgent):+ def predict_batch(self, states, questions, batch_size=None):+ self.calls.append(("predict_batch", tuple(states), batch_size))+ return [self._answer(questions) for _ in states]+++# ---------------------------------------------------------------- predict contract+agent = _FakeAgent("english-ckpt")+runner = OnnxRunner(agent)+Q = {"intent": {"type": "choice", "instructions": "i", "criteria": {"billing": "b"}}}+res = runner.predict("a state", Q)+check("predict/returns the agent answer", res["answers"]["intent"]["choice"], "billing")+check("predict/forwards state to the agent", agent.calls[-1], ("predict", "a state"))+runner.predict("s2", Q, model="english-ckpt")+check_true("predict/matching per-example model is accepted", True)++try:+ runner.predict("s3", Q, model="multilingual")+ check_true("predict/foreign model raises EvalError", False, "no exception")+except EvalError as exc:+ msg = str(exc)+ check_true("predict/foreign model raises EvalError", True)+ check_true("predict/error names the served checkpoint", "english-ckpt" in msg, msg)+ check_true("predict/error names the asked-for checkpoint", "multilingual" in msg, msg)++# ---------------------------------------------------------------- predict_batch contract+agent_nobatch = _FakeAgent("ckpt")+OnnxRunner(agent_nobatch).predict_batch(["a", "b", "c"], Q)+check("predict_batch/falls back to one predict per state",+ [c[1] for c in agent_nobatch.calls], ["a", "b", "c"])++bagent = _BatchAgent("ckpt")+out = OnnxRunner(bagent).predict_batch(["a", "b"], Q, batch_size=2)+check("predict_batch/delegates once when the agent can batch",+ bagent.calls, [("predict_batch", ("a", "b"), 2)])+check("predict_batch/delegated result passes through", len(out), 2)+try:+ OnnxRunner(bagent).predict_batch(["a"], Q, model="other")+ check_true("predict_batch/foreign model raises too", False, "no exception")+except EvalError:+ check_true("predict_batch/foreign model raises too", True)++# ---------------------------------------------------------------- CLI wiring+args = evals_cli._build_parser().parse_args(["run", "d.jsonl", "--onnx", "m.onnx"])+check("cli/--onnx parses on the run subcommand", args.onnx, "m.onnx")+args = evals_cli._build_parser().parse_args(["run", "d.jsonl"])+check("cli/--onnx defaults to None (torch Router path unchanged)", args.onnx, None)++tmp = tempfile.mkdtemp(prefix="laya_evals_onnx_")+dataset = os.path.join(tmp, "data.jsonl")+with open(dataset, "w") as f:+ f.write(json.dumps({"state": "charged twice, refund me",+ "questions": Q, "expected": {"intent": "billing"}}) + "\n")+ f.write(json.dumps({"state": "billed again after cancelling",+ "questions": Q, "expected": {"intent": "billing"}}) + "\n")+report_path = os.path.join(tmp, "report.json")++import laya.onnx_agent as onnx_agent_module # noqa: E402 (numpy-only at import time)++_real_onnx_agent = onnx_agent_module.ONNXAgent+try:+ onnx_agent_module.ONNXAgent = _FakeAgent+ rc = evals_cli.main(["run", dataset, "--onnx", os.path.join(tmp, "laya.onnx"),+ "--model", "english-ckpt", "--json", report_path])+ check("cli/run --onnx exits 0 on a passing dataset", rc, 0)+ with open(report_path) as f:+ report = json.load(f)+ check("cli/report records the onnx export path",+ report["config"]["onnx"], os.path.join(tmp, "laya.onnx"))+ check("cli/report keeps the forced checkpoint", report["config"]["model"], "english-ckpt")+ check("cli/metrics are computed from the ONNX answers",+ report["overall"]["choice_accuracy"], 1.0)++ # --model omitted: the export's checkpoint defaults to the english bundle repo.+ rc = evals_cli.main(["run", dataset, "--onnx", "laya.onnx"])+ check("cli/run without --model still scores", rc, 0)++ # batching reaches the runner's predict_batch through the harness+ onnx_agent_module.ONNXAgent = _BatchAgent+ rc = evals_cli.main(["run", dataset, "--onnx", "laya.onnx", "--model", "ckpt",+ "--batch-size", "2"])+ check("cli/--batch-size runs the ONNX batch path", rc, 0)++ # a per-example model the single-checkpoint agent does not serve is an error, not a no-op+ # (no --model here: --model would authoritatively rewrite every row and mask the mismatch)+ mismatch = os.path.join(tmp, "mismatch.jsonl")+ with open(mismatch, "w") as f:+ row = json.loads(open(dataset).readline())+ row["model"] = "some-other-checkpoint"+ f.write(json.dumps(row) + "\n")+ rc = evals_cli.main(["run", mismatch, "--onnx", "laya.onnx"])+ check("cli/foreign per-example model exits 1", rc, 1)+finally:+ onnx_agent_module.ONNXAgent = _real_onnx_agent++# ---------------------------------------------------------------- torch path unchanged+check("api/RouterRunner still exists beside OnnxRunner", hasattr(evals_cli, "RouterRunner"), True)++# ---------------------------------------------------------------- report+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))+for f in FAIL:+ print(" FAIL " + f)+sys.exit(1 if FAIL else 0)