Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(evals): accept --calibration on laya-evals run --onnx

naveenprasaath23e969bef0a7cc
diff --git a/docs/evals.md b/docs/evals.md
index 0e11aefc..68ea71a6 100644
--- a/docs/evals.md
+++ b/docs/evals.md
@@ -46,8 +46,10 @@ short name like `english`, since there is no Router on this path (default
whose `model` field names a different one fails with a clear error rather than being silently
answered by the wrong model; `--device` does not apply. `--batch-size` uses the agent's batch
API when it has one and falls back to one call per state otherwise; `--sort-by-length` is forwarded
-to that batch API, which the per-state fallback has no group to reorder. The report's `config` block
-records the `onnx` path.
+to that batch API, which the per-state fallback has no group to reorder. Pass `--calibration PATH`
+to load a fitted calibration map onto `ONNXAgent`, so calibration gates such as `--max-ece`
+evaluate against calibrated probabilities. The report's `config` block records the `onnx` path
+and `calibration` path (when set).
Measured on `research/evals/fixture.jsonl` (12 labelled rows, English checkpoint, CPU):
diff --git a/laya/evals_cli.py b/laya/evals_cli.py
index a58375d0..c63b0a99 100644
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@ -145,6 +145,8 @@ def _build_parser() -> argparse.ArgumentParser:
help="evaluate an ONNX export through ONNXAgent instead of the torch Router; "
"--model then names the checkpoint directory or Hub id the export came "
"from (default convaiinnovations/laya)")
+ run.add_argument("--calibration", metavar="PATH",
+ help="path to a JSON calibration map for ONNXAgent (requires --onnx)")
run.add_argument("--revision", action="append", metavar="SHA | NAME=SHA",
help="pin the checkpoint commit: a bare SHA applies to every checkpoint this "
"run loads, NAME=SHA pins one (repeatable). Unpinned runs fetch the "
@@ -270,9 +272,11 @@ def _cmd_run(args) -> int:
if revisions:
raise EvalError("--revision NAME=SHA needs the Router; with --onnx pass one bare SHA")
agent = ONNXAgent(args.model or "convaiinnovations/laya", onnx_path=args.onnx,
- revision=revision)
+ revision=revision, calibration=args.calibration)
runner: Any = OnnxRunner(agent)
else:
+ if args.calibration:
+ raise EvalError("--calibration requires --onnx; for the torch Router calibrate via fit_temperatures")
import laya
try:
pins: Dict[str, Any] = {}
@@ -297,6 +301,8 @@ def _cmd_run(args) -> int:
"dataset_sha256": evals.file_fingerprint(args.dataset)}
if args.onnx:
config["onnx"] = args.onnx
+ if args.calibration:
+ config["calibration"] = args.calibration
if extra:
config["score_within"] = [evaluator.tolerance for evaluator in extra]
report = evals.evaluate(runner, dataset, evaluators=evals.default_evaluators() + extra,
diff --git a/tests/test_evals_onnx.py b/tests/test_evals_onnx.py
index a306b0ef..7508f3ea 100644
--- a/tests/test_evals_onnx.py
+++ b/tests/test_evals_onnx.py
@@ -43,6 +43,7 @@ class _FakeAgent:
def __init__(self, model_id_or_path, onnx_path=None, **kw):
self.model_id = model_id_or_path
self.onnx_path = onnx_path
+ self.calibration = kw.get("calibration")
self.calls = []
def _answer(self, questions):
@@ -122,6 +123,10 @@ args = evals_cli._build_parser().parse_args(["run", "d.jsonl", "--onnx", "m.onnx
check("cli/--onnx parses on the run subcommand", args.onnx, "m.onnx")
args = evals_cli._build_parser().parse_args(["run", "d.jsonl"])
check("cli/--onnx defaults to None (torch Router path unchanged)", args.onnx, None)
+args = evals_cli._build_parser().parse_args(["run", "d.jsonl", "--onnx", "m.onnx", "--calibration", "c.json"])
+check("cli/--calibration parses on the run subcommand", args.calibration, "c.json")
+args = evals_cli._build_parser().parse_args(["run", "d.jsonl"])
+check("cli/--calibration defaults to None", args.calibration, None)
tmp = tempfile.mkdtemp(prefix="laya_evals_onnx_")
dataset = os.path.join(tmp, "data.jsonl")
@@ -144,10 +149,27 @@ try:
report = json.load(f)
check("cli/report records the onnx export path",
report["config"]["onnx"], os.path.join(tmp, "laya.onnx"))
+ check("cli/report omits calibration when unset", "calibration" not in report["config"], True)
check("cli/report keeps the forced checkpoint", report["config"]["model"], "english-ckpt")
check("cli/metrics are computed from the ONNX answers",
report["overall"]["choice_accuracy"], 1.0)
+ # --calibration without --onnx is rejected before model load
+ rc_no_onnx = evals_cli.main(["run", dataset, "--calibration", "calib.json"])
+ check("cli/--calibration without --onnx exits 2", rc_no_onnx, 2)
+
+ # --calibration with --onnx is forwarded to ONNXAgent and recorded in report config
+ calib_file = os.path.join(tmp, "calib.json")
+ calib_report_path = os.path.join(tmp, "report_calib.json")
+ rc = evals_cli.main(["run", dataset, "--onnx", os.path.join(tmp, "laya.onnx"),
+ "--model", "english-ckpt", "--calibration", calib_file,
+ "--json", calib_report_path])
+ check("cli/run --onnx --calibration exits 0", rc, 0)
+ with open(calib_report_path) as f:
+ calib_report = json.load(f)
+ check("cli/report records the calibration path",
+ calib_report["config"]["calibration"], calib_file)
+
# --model omitted: the export's checkpoint defaults to the english bundle repo.
rc = evals_cli.main(["run", dataset, "--onnx", "laya.onnx"])
check("cli/run without --model still scores", rc, 0)