Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals-cli): record the ONNX run's checkpoint commit in the report

Aashisha3f62b768c1e
diff --git a/laya/evals_cli.py b/laya/evals_cli.py
index 0d8ff5ee..2a3a71d8 100644
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@ -401,8 +401,16 @@ def _cmd_run(args) -> int:
# the run: `preload=False` means no checkpoint is resident before the first row.
# `loaded_revisions` reports the commit each resident agent came from -- the pin when there is
# one, the commit the default branch resolved to when there is not, None for a local path.
- if router is not None:
- report.config = dict(report.config, revisions=getattr(router, "loaded_revisions", {}))
+ # The ONNX arm answers from one checkpoint and `ONNXAgent` resolves the same fact for it
+ # (`self.revision`, `laya/onnx_agent.py`), so the block is written there too: `--revision`
+ # promises "the report records the commit that answered either way", and a `revisions` block
+ # that only ever appears on the torch path does not keep it -- nor does the config table in
+ # `docs/evals.md`, which documents `revisions` for every `run`.
+ if args.onnx:
+ revisions = {agent.model_id: getattr(agent, "revision", None)}
+ else:
+ revisions = getattr(router, "loaded_revisions", {})
+ report.config = dict(report.config, revisions=revisions)
if extra:
_warn_no_value(extra, report)
diff --git a/tests/test_evals_onnx.py b/tests/test_evals_onnx.py
index 7508f3ea..ec268c39 100644
--- a/tests/test_evals_onnx.py
+++ b/tests/test_evals_onnx.py
@@ -22,6 +22,12 @@ from laya.evals_cli import OnnxRunner # noqa: E402
PASS, FAIL = [], []
+# Two different commits, on purpose: a pin and the commit an unpinned default branch resolved to.
+# One value for both would let a check pass no matter which arm of `ONNXAgent`'s
+# `snapshot_revision(model_dir) or revision` answered.
+PINNED = "1" * 40
+RESOLVED_REVISION = "2" * 40
+
def check(name, got, want):
if got == want:
@@ -44,6 +50,12 @@ class _FakeAgent:
self.model_id = model_id_or_path
self.onnx_path = onnx_path
self.calibration = kw.get("calibration")
+ # `ONNXAgent` sets `self.revision` from the snapshot it downloaded -- the commit the
+ # default branch resolved to, or the pin when the caller pinned one -- and leaves it None
+ # when the source is a local directory that already exists. The stand-in models all three
+ # arms, because the report block is read from this attribute.
+ self.revision = None if os.path.exists(model_id_or_path) else (
+ kw.get("revision") or RESOLVED_REVISION)
self.calls = []
def _answer(self, questions):
@@ -151,6 +163,37 @@ try:
report["config"]["onnx"], os.path.join(tmp, "laya.onnx"))
check("cli/report omits calibration when unset", "calibration" not in report["config"], True)
check("cli/report keeps the forced checkpoint", report["config"]["model"], "english-ckpt")
+
+ # `--revision`'s help promises "the report records the commit that answered either way", and
+ # docs/evals.md lists `revisions` as a block of every `run`. Only the torch Router kept it:
+ # the block was read from `router.loaded_revisions` under `if router is not None`, so an ONNX
+ # report carried no `revisions` key at all -- an unattributable artifact from a CLI that says
+ # it is attributable. The three shapes `ONNXAgent.revision` takes each get a check.
+ pinned_path = os.path.join(tmp, "pinned.json")
+ rc = evals_cli.main(["run", dataset, "--onnx", os.path.join(tmp, "laya.onnx"),
+ "--model", "english-ckpt", "--revision", PINNED, "--json", pinned_path])
+ check("cli/run --onnx --revision exits 0", rc, 0)
+ with open(pinned_path) as f:
+ pinned_report = json.load(f)
+ check("cli/--onnx records the pinned commit against the checkpoint it serves",
+ pinned_report["config"]["revisions"], {"english-ckpt": PINNED})
+ check("cli/an unpinned ONNX run records the commit the default branch resolved to",
+ report["config"]["revisions"], {"english-ckpt": RESOLVED_REVISION})
+
+ # A local checkpoint directory has no commit to name, and `ONNXAgent` leaves `revision` None.
+ # The block still says that, rather than leaving the reader to tell "ran from a local path"
+ # apart from "the harness forgot to record it".
+ local_ckpt = os.path.join(tmp, "ckptdir")
+ os.makedirs(local_ckpt)
+ local_path = os.path.join(tmp, "local.json")
+ rc = evals_cli.main(["run", dataset, "--onnx", os.path.join(tmp, "laya.onnx"),
+ "--model", local_ckpt, "--json", local_path])
+ check("cli/run --onnx from a local checkpoint exits 0", rc, 0)
+ with open(local_path) as f:
+ local_report = json.load(f)
+ check("cli/a local ONNX checkpoint is recorded as having no commit",
+ local_report["config"]["revisions"], {local_ckpt: None})
+
check("cli/metrics are computed from the ONNX answers",
report["overall"]["choice_accuracy"], 1.0)