Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #588 from aashish254/feat-evals-revision

Nandakishor691d0fcf2d20
diff --cc docs/evals.md
index 9b9ec7f8,2d722be3..56af9c0e
--- a/docs/evals.md
+++ b/docs/evals.md
@@@ -138,8 -88,14 +138,15 @@@ Two CI surfaces use this
PR.
The harness is deterministic for a fixed checkpoint revision, so a report is reproducible.
-`run` records the dataset, model and device in the report's `config` block, plus `revisions`:
-the commit each checkpoint that answered was actually loaded from. `--revision <SHA>` pins that
-commit for every checkpoint the run loads, and `--revision english=<SHA>` pins one checkpoint
-(repeatable) — which is the form an auto-routing run wants, since the three checkpoints are three
-repositories and one commit cannot exist in all of them. Left unpinned, the run takes the
-checkpoint's default branch and the report still says which commit answered, so a baseline drift
-can be attributed to the weights or to the code. `laya/revisions.py` publishes reviewed commit
-SHAs in `PINNED_REVISIONS` for callers who want to opt in.
+`run` records the dataset, model and device, plus the [timing](#batching-and-timing) facts of the
- run, in the report's `config` block.
++run, in the report's `config` block, and `revisions`: the commit each checkpoint that answered was
++actually loaded from. `--revision <SHA>` pins that commit for every checkpoint the run loads, and
++`--revision english=<SHA>` pins one checkpoint (repeatable) — which is the form an auto-routing run
++wants, since the three checkpoints are three repositories and one commit cannot exist in all of
++them. Left unpinned, the run takes the checkpoint's default branch and the report still says which
++commit answered, so a baseline drift can be attributed to the weights or to the code.
++`laya/revisions.py` publishes reviewed commit SHAs in `PINNED_REVISIONS` for callers who want to
++opt in. With `--onnx`, only a bare `--revision <SHA>` applies, to the config and tokenizer download.
## Adding the real labelled set
diff --cc laya/evals_cli.py
index 7256dda1,0d8ea284..4fef8794
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@@ -90,10 -86,11 +117,15 @@@ def _build_parser() -> argparse.Argumen
run.add_argument("dataset")
run.add_argument("--model", help="force a checkpoint instead of auto-routing")
run.add_argument("--device", help="torch device, e.g. cpu or cuda")
+ run.add_argument("--onnx", metavar="PATH",
+ help="evaluate an ONNX export through ONNXAgent instead of the torch Router; "
+ "--model then names the checkpoint directory or Hub id the export came "
+ "from (default convaiinnovations/laya)")
+ run.add_argument("--revision", action="append", metavar="SHA | NAME=SHA",
+ help="pin the checkpoint commit: a bare SHA applies to every checkpoint this "
+ "run loads, NAME=SHA pins one (repeatable). Unpinned runs fetch the "
+ "checkpoint's default branch; the report records the commit that answered "
+ "either way")
run.add_argument("--batch-size", type=int, help="examples per forward pass when questions match")
run.add_argument("--on-error", choices=("fail", "skip"), default="fail")
run.add_argument("--baseline", help="a baseline report JSON to compare against")
@@@ -164,20 -154,25 +196,42 @@@ def _cmd_run(args) -> int
if args.model:
for example in dataset.examples: # --model is authoritative over per-row model
example.model = args.model
+ revision, revisions = _parse_revisions(args.revision)
- try:
- router = laya.Router(device=args.device, preload=False, revision=revision,
- revisions=revisions)
- except ValueError as exc:
- # `Router` already rejects a checkpoint name it does not know -- normalising case,
- # surrounding spaces and aliases on the way -- with the option list in the message. This
- # only turns that into `laya-evals: unknown model 'englishg'; choose one of …` instead of
- # a traceback, so a mistyped pin fails as a usage error and never starts a download.
- raise EvalError(str(exc)) from exc
- runner: Any = RouterRunner(router)
++ router = None
+ if args.onnx:
+ from .onnx_agent import ONNXAgent
+ # The export was produced from one checkpoint; --model names it (the ONNXAgent load
- # needs its config and tokenizer), so the default is the english bundle repo.
- agent = ONNXAgent(args.model or "convaiinnovations/laya", onnx_path=args.onnx)
++ # needs its config and tokenizer), so the default is the english bundle repo. A bare
++ # --revision pins that checkpoint's config and tokenizer download.
++ if revisions:
++ raise EvalError("--revision NAME=SHA needs the Router; with --onnx pass one bare SHA")
++ agent = ONNXAgent(args.model or "convaiinnovations/laya", onnx_path=args.onnx,
++ revision=revision)
+ runner: Any = OnnxRunner(agent)
+ else:
+ import laya
- runner = RouterRunner(laya.Router(device=args.device, preload=False))
++ try:
++ router = laya.Router(device=args.device, preload=False, revision=revision,
++ revisions=revisions)
++ except ValueError as exc:
++ # `Router` already rejects a checkpoint name it does not know -- normalising case,
++ # surrounding spaces and aliases on the way -- with the option list in the message.
++ # This only turns that into `laya-evals: unknown model 'englishg'; choose one of …`
++ # instead of a traceback, so a mistyped pin fails as a usage error and never starts a
++ # download.
++ raise EvalError(str(exc)) from exc
++ runner = RouterRunner(router)
config = {"dataset": args.dataset, "model": args.model, "device": args.device}
+ if args.onnx:
+ config["onnx"] = args.onnx
report = evals.evaluate(runner, dataset, batch_size=args.batch_size,
on_error=args.on_error, config=config)
+ # Which commit answered belongs in the artifact a baseline is, and it can only be read after
+ # the run: `preload=False` means no checkpoint is resident before the first row.
+ # `loaded_revisions` reports the commit each resident agent came from -- the pin when there is
+ # one, the commit the default branch resolved to when there is not, None for a local path.
- report.config = dict(report.config, revisions=router.loaded_revisions)
++ if router is not None:
++ report.config = dict(report.config, revisions=router.loaded_revisions)
mins = _parse_pairs(args.min)
maxs = _parse_pairs(args.max)