Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(evals): accept --calibration on laya-evals run --onnx
naveenprasaath23e969bef0a7cc
diff --git a/docs/evals.md b/docs/evals.mdindex 0e11aefc..68ea71a6 100644--- a/docs/evals.md+++ b/docs/evals.md@@ -46,8 +46,10 @@ short name like `english`, since there is no Router on this path (defaultwhose `model` field names a different one fails with a clear error rather than being silentlyanswered by the wrong model; `--device` does not apply. `--batch-size` uses the agent's batchAPI when it has one and falls back to one call per state otherwise; `--sort-by-length` is forwarded-to that batch API, which the per-state fallback has no group to reorder. The report's `config` block-records the `onnx` path.+to that batch API, which the per-state fallback has no group to reorder. Pass `--calibration PATH`+to load a fitted calibration map onto `ONNXAgent`, so calibration gates such as `--max-ece`+evaluate against calibrated probabilities. The report's `config` block records the `onnx` path+and `calibration` path (when set).Measured on `research/evals/fixture.jsonl` (12 labelled rows, English checkpoint, CPU):diff --git a/laya/evals_cli.py b/laya/evals_cli.pyindex a58375d0..c63b0a99 100644--- a/laya/evals_cli.py+++ b/laya/evals_cli.py@@ -145,6 +145,8 @@ def _build_parser() -> argparse.ArgumentParser:help="evaluate an ONNX export through ONNXAgent instead of the torch Router; ""--model then names the checkpoint directory or Hub id the export came ""from (default convaiinnovations/laya)")+ run.add_argument("--calibration", metavar="PATH",+ help="path to a JSON calibration map for ONNXAgent (requires --onnx)")run.add_argument("--revision", action="append", metavar="SHA | NAME=SHA",help="pin the checkpoint commit: a bare SHA applies to every checkpoint this ""run loads, NAME=SHA pins one (repeatable). Unpinned runs fetch the "@@ -270,9 +272,11 @@ def _cmd_run(args) -> int:if revisions:raise EvalError("--revision NAME=SHA needs the Router; with --onnx pass one bare SHA")agent = ONNXAgent(args.model or "convaiinnovations/laya", onnx_path=args.onnx,- revision=revision)+ revision=revision, calibration=args.calibration)runner: Any = OnnxRunner(agent)else:+ if args.calibration:+ raise EvalError("--calibration requires --onnx; for the torch Router calibrate via fit_temperatures")import layatry:pins: Dict[str, Any] = {}@@ -297,6 +301,8 @@ def _cmd_run(args) -> int:"dataset_sha256": evals.file_fingerprint(args.dataset)}if args.onnx:config["onnx"] = args.onnx+ if args.calibration:+ config["calibration"] = args.calibrationif extra:config["score_within"] = [evaluator.tolerance for evaluator in extra]report = evals.evaluate(runner, dataset, evaluators=evals.default_evaluators() + extra,diff --git a/tests/test_evals_onnx.py b/tests/test_evals_onnx.pyindex a306b0ef..7508f3ea 100644--- a/tests/test_evals_onnx.py+++ b/tests/test_evals_onnx.py@@ -43,6 +43,7 @@ class _FakeAgent:def __init__(self, model_id_or_path, onnx_path=None, **kw):self.model_id = model_id_or_pathself.onnx_path = onnx_path+ self.calibration = kw.get("calibration")self.calls = []def _answer(self, questions):@@ -122,6 +123,10 @@ args = evals_cli._build_parser().parse_args(["run", "d.jsonl", "--onnx", "m.onnxcheck("cli/--onnx parses on the run subcommand", args.onnx, "m.onnx")args = evals_cli._build_parser().parse_args(["run", "d.jsonl"])check("cli/--onnx defaults to None (torch Router path unchanged)", args.onnx, None)+args = evals_cli._build_parser().parse_args(["run", "d.jsonl", "--onnx", "m.onnx", "--calibration", "c.json"])+check("cli/--calibration parses on the run subcommand", args.calibration, "c.json")+args = evals_cli._build_parser().parse_args(["run", "d.jsonl"])+check("cli/--calibration defaults to None", args.calibration, None)tmp = tempfile.mkdtemp(prefix="laya_evals_onnx_")dataset = os.path.join(tmp, "data.jsonl")@@ -144,10 +149,27 @@ try:report = json.load(f)check("cli/report records the onnx export path",report["config"]["onnx"], os.path.join(tmp, "laya.onnx"))+ check("cli/report omits calibration when unset", "calibration" not in report["config"], True)check("cli/report keeps the forced checkpoint", report["config"]["model"], "english-ckpt")check("cli/metrics are computed from the ONNX answers",report["overall"]["choice_accuracy"], 1.0)+ # --calibration without --onnx is rejected before model load+ rc_no_onnx = evals_cli.main(["run", dataset, "--calibration", "calib.json"])+ check("cli/--calibration without --onnx exits 2", rc_no_onnx, 2)++ # --calibration with --onnx is forwarded to ONNXAgent and recorded in report config+ calib_file = os.path.join(tmp, "calib.json")+ calib_report_path = os.path.join(tmp, "report_calib.json")+ rc = evals_cli.main(["run", dataset, "--onnx", os.path.join(tmp, "laya.onnx"),+ "--model", "english-ckpt", "--calibration", calib_file,+ "--json", calib_report_path])+ check("cli/run --onnx --calibration exits 0", rc, 0)+ with open(calib_report_path) as f:+ calib_report = json.load(f)+ check("cli/report records the calibration path",+ calib_report["config"]["calibration"], calib_file)+# --model omitted: the export's checkpoint defaults to the english bundle repo.rc = evals_cli.main(["run", dataset, "--onnx", "laya.onnx"])check("cli/run without --model still scores", rc, 0)