Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #813 from aashish254/feat-evals-help-strings

Nandakishor298e443eba08
diff --cc laya/evals_cli.py
index bec4a8d3,d5e00c82..4139570a
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@@ -171,14 -159,10 +175,17 @@@ def _build_parser() -> argparse.Argumen
help="with --batch-size N where 1 < N < the run, group similarly sized "
"examples into the same forward pass so each pads to a shorter maximum; "
"scores the same answers, in the same order")
+ run.add_argument("--min-confidence", dest="min_confidence", type=float, metavar="THRESHOLD",
+ help="abstention threshold on `answer_confidence` (#361): answers below it "
+ "come back abstained, so the run scores the policy at that threshold "
+ "rather than the raw argmax. Accepted range is core's -- "
+ "`laya.confidence.check_min_confidence` -- not a copy of it here, and a "
+ "runner that predates the gate is refused with a named error rather "
+ "than silently scored without it")
- run.add_argument("--on-error", choices=("fail", "skip"), default="fail")
+ run.add_argument("--on-error", choices=("fail", "skip"), default="fail",
+ help="'fail' (the default) stops the run when a runner call raises; 'skip' "
+ "lists every row it could not score under the report's config.errored "
+ "and reports the metrics for the rows that returned")
run.add_argument("--baseline", help="a baseline report JSON to compare against")
run.add_argument("--tolerance", action="append", metavar="METRIC=VALUE",
help="allowed absolute drift from the baseline; repeatable")
@@@ -199,12 -181,12 +206,14 @@@
run.add_argument("--markdown", dest="markdown_out", help="write a Markdown summary here")
compare = sub.add_parser("compare", help="compare a report JSON against a baseline")
- compare.add_argument("report")
- compare.add_argument("--baseline", required=True)
+ compare.add_argument("report", help="a report JSON written by `laya-evals run --json`")
+ compare.add_argument("--baseline", required=True,
+ help="the reviewed report JSON this one is checked against; its dataset "
+ "and question fingerprints must match")
compare.add_argument("--tolerance", action="append", metavar="METRIC=VALUE",
help="allowed absolute drift; repeatable")
+ compare.add_argument("--gate-policy", metavar="FILE",
+ help="apply opt-in per-slice quality rules from a JSON policy")
return parser