Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
Merge pull request #813 from aashish254/feat-evals-help-strings
Nandakishor298e443eba08
diff --cc laya/evals_cli.pyindex bec4a8d3,d5e00c82..4139570a--- a/laya/evals_cli.py+++ b/laya/evals_cli.py@@@ -171,14 -159,10 +175,17 @@@ def _build_parser() -> argparse.Argumenhelp="with --batch-size N where 1 < N < the run, group similarly sized ""examples into the same forward pass so each pads to a shorter maximum; ""scores the same answers, in the same order")+ run.add_argument("--min-confidence", dest="min_confidence", type=float, metavar="THRESHOLD",+ help="abstention threshold on `answer_confidence` (#361): answers below it "+ "come back abstained, so the run scores the policy at that threshold "+ "rather than the raw argmax. Accepted range is core's -- "+ "`laya.confidence.check_min_confidence` -- not a copy of it here, and a "+ "runner that predates the gate is refused with a named error rather "+ "than silently scored without it")- run.add_argument("--on-error", choices=("fail", "skip"), default="fail")+ run.add_argument("--on-error", choices=("fail", "skip"), default="fail",+ help="'fail' (the default) stops the run when a runner call raises; 'skip' "+ "lists every row it could not score under the report's config.errored "+ "and reports the metrics for the rows that returned")run.add_argument("--baseline", help="a baseline report JSON to compare against")run.add_argument("--tolerance", action="append", metavar="METRIC=VALUE",help="allowed absolute drift from the baseline; repeatable")@@@ -199,12 -181,12 +206,14 @@@run.add_argument("--markdown", dest="markdown_out", help="write a Markdown summary here")compare = sub.add_parser("compare", help="compare a report JSON against a baseline")- compare.add_argument("report")- compare.add_argument("--baseline", required=True)+ compare.add_argument("report", help="a report JSON written by `laya-evals run --json`")+ compare.add_argument("--baseline", required=True,+ help="the reviewed report JSON this one is checked against; its dataset "+ "and question fingerprints must match")compare.add_argument("--tolerance", action="append", metavar="METRIC=VALUE",help="allowed absolute drift; repeatable")+ compare.add_argument("--gate-policy", metavar="FILE",+ help="apply opt-in per-slice quality rules from a JSON policy")return parser