Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(evals): let laya-evals run ask for a tolerance metric
Aashishf8a485583080
diff --git a/docs/evals.md b/docs/evals.mdindex 3660ec3a..0503245a 100644--- a/docs/evals.md+++ b/docs/evals.md@@ -15,7 +15,7 @@ laya-evals validate research/evals/fixture.jsonl# score a labelled set on one checkpoint, with thresholds and a baselinelaya-evals run data.jsonl --model english --device cpu \- --min-accuracy 0.8 --max-ece 0.05 --slice language \+ --min-accuracy 0.8 --max-ece 0.05 --score-within 0.25 --slice language \--json report.json --markdown report.md# compare a saved report to a baseline@@ -59,7 +59,14 @@ Each metric is computed per answer where it applies and aggregated over the data| `latency_p50_ms`, `latency_p95_ms` | per request | wall time, informational |Add `ScoreWithin(0.25)` to the evaluator list for a tolerance metric; the default set is-`choice_accuracy`, `noul_accuracy`, `score_mae`, `mean_confidence`, plus `ece`.+`choice_accuracy`, `noul_accuracy`, `score_mae`, `mean_confidence`, plus `ece`. From the CLI the+same thing is one flag: `laya-evals run data.jsonl --score-within 0.25` reports `score_within_0.25`+beside the defaults, and the flag repeats, so `--score-within 0.25 --score-within 0.5` reports both.++A tolerance metric needs a `score` answer with a numeric label, so on a dataset without one it has+no value: `run` names the metric it could not compute instead of publishing a silent zero, and a+`--min` / `--max` gate naming that metric fails as missing. The tolerances a run was asked for are+recorded in the report's `config` block, so a reviewed baseline says which columns it expects.## Slicesdiff --git a/laya/evals_cli.py b/laya/evals_cli.pyindex e656eba6..ee5a6fb8 100644--- a/laya/evals_cli.py+++ b/laya/evals_cli.py@@ -5,6 +5,7 @@ Exit codes: 0 on success, 1 when a threshold or a baseline tolerance fails, 2 onlaya-evals validate research/evals/fixture.jsonllaya-evals run research/evals/fixture.jsonl --model english --min-accuracy 0.8 --max-ece 0.1laya-evals run data.jsonl --baseline baseline.json --tolerance choice_accuracy=0.02 --json out.json+ laya-evals run data.jsonl --score-within 0.25 --min score_within_0.25=0.9`laya eval ...` dispatches here from the main CLI, so both spellings work."""@@ -12,6 +13,7 @@ from __future__ import annotationsimport argparseimport json+import mathimport sysfrom typing import Any, Dict, List, Optional, Sequence@@ -47,6 +49,20 @@ def _parse_pairs(pairs: Optional[Sequence[str]]) -> Dict[str, float]:return out+def _score_within(tolerances: Optional[Sequence[float]]) -> List[evals.Evaluator]:+ """Build the `ScoreWithin` evaluators `--score-within` asks for, in the order given.++ A non-finite or negative tolerance would name a metric that is always 1.0 or always 0.0, and a+ `--min` gate on that metric would then decide nothing, so the CLI rejects them up front.+ """+ out: List[evals.Evaluator] = []+ for tolerance in tolerances or []:+ if not math.isfinite(tolerance) or tolerance < 0.0:+ raise EvalError("--score-within needs a finite tolerance >= 0, got %g" % tolerance)+ out.append(evals.ScoreWithin(tolerance))+ return out++def _build_parser() -> argparse.ArgumentParser:parser = argparse.ArgumentParser(prog="laya-evals",description="Evaluate a Laya checkpoint on a labelled dataset.")@@ -67,6 +83,10 @@ def _build_parser() -> argparse.ArgumentParser:run.add_argument("--min-accuracy", type=float,help="minimum accuracy (choice, else noul) for the whole dataset")run.add_argument("--max-ece", type=float, help="maximum expected calibration error")+ run.add_argument("--score-within", dest="score_within", type=float, action="append",+ metavar="TOL",+ help="also report score_within_TOL, the fraction of score answers within TOL "+ "of the label; repeatable, and added to the default metrics")run.add_argument("--min", action="append", metavar="METRIC=VALUE", help="minimum for any metric")run.add_argument("--max", action="append", metavar="METRIC=VALUE", help="maximum for any metric")run.add_argument("--slice", action="append", choices=("language", "model", "qid", "tag"),@@ -115,6 +135,22 @@ def _cmd_validate(args) -> int:return 0+def _warn_no_value(extra: Sequence[evals.Evaluator], report: evals.EvalReport) -> None:+ """Say so when a metric the caller asked for scored nothing.++ `evaluate` drops a metric no answer applied to, so a requested `--score-within` over a+ choice-only dataset would otherwise change the command and not the report.+ """+ missing = [evaluator.name for evaluator in extra if evaluator.name not in report.overall]+ if not missing:+ return+ score_cases = sum(1 for case in report.cases if (case.get("answer") or {}).get("type") == "score")+ for name in missing:+ print("laya-evals: %s has no value: %d of %d answered case(s) are score answers, and the "+ "metric needs one whose label is a number. A threshold naming it reports it missing."+ % (name, score_cases, len(report.cases)), file=sys.stderr)++def _cmd_run(args) -> int:import laya@@ -122,10 +158,16 @@ def _cmd_run(args) -> int:if args.model:for example in dataset.examples: # --model is authoritative over per-row modelexample.model = args.model+ extra = _score_within(args.score_within) # before the checkpoint loads: a bad flag is cheaprunner: Any = RouterRunner(laya.Router(device=args.device, preload=False))config = {"dataset": args.dataset, "model": args.model, "device": args.device}- report = evals.evaluate(runner, dataset, batch_size=args.batch_size,- on_error=args.on_error, config=config)+ if extra:+ config["score_within"] = [evaluator.tolerance for evaluator in extra]+ report = evals.evaluate(runner, dataset, evaluators=evals.default_evaluators() + extra,+ batch_size=args.batch_size, on_error=args.on_error, config=config)++ if extra:+ _warn_no_value(extra, report)mins = _parse_pairs(args.min)maxs = _parse_pairs(args.max)diff --git a/tests/test_evals.py b/tests/test_evals.pyindex 0e9ff2d7..8f7d50b4 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -3,6 +3,7 @@Run: python -m pytest tests/test_evals.py -q"""import json+import reimport pytest@@ -36,6 +37,10 @@ def noul_answer(prob):return {"type": "noul", "noul": prob, "confidence": max(prob, 1 - prob)}+def score_answer(value, confidence=0.8):+ return {"type": "score", "score": value, "confidence": confidence}++class StubRunner:"""Returns fixed answers per state, so evaluation is deterministic and weight-free."""@@ -229,3 +234,145 @@ def test_cli_rejects_a_malformed_tolerance():with pytest.raises(EvalError):evals_cli._parse_pairs(["choice_accuracy"])+++# ------------------------------------------------------------------ CLI run+# One score row inside 0.25 of its label, one 0.3 away (inside 0.5 but not 0.25), plus a choice and+# an noul row so every default metric is in the report too.+RUN_ROWS = [+ {"state": "near", "questions": QSCORE, "expected": {"quality": 4}},+ {"state": "far", "questions": QSCORE, "expected": {"quality": 4}},+ {"state": "intent", "questions": Q, "expected": {"intent": "a"}},+ {"state": "flag", "questions": QNOUL, "expected": {"flag": True}},+]+RUN_ANSWERS = {+ "near": {"quality": score_answer(4.0)},+ "far": {"quality": score_answer(3.7)},+ "intent": {"intent": choice_answer("a")},+ "flag": {"flag": noul_answer(0.9)},+}+++def _patch_router(monkeypatch, answers=None):+ """Replace the checkpoint-loading `Router`, so `laya-evals run` needs no weights.++ Returns the list the stand-in appends to when it is constructed, so a test can show a bad flag+ fails before anything loads.+ """+ import laya++ built: list = []++ class FakeRouter:+ def __init__(self, device=None, preload=False):+ built.append(device)++ def predict(self, state, questions, model=None):+ return {"model": model or "stub", "answers": (answers or RUN_ANSWERS)[state]}++ monkeypatch.setattr(laya, "Router", FakeRouter)+ return built+++def _overall(stdout):+ return {name: float(value) for name, value in+ re.findall(r"^(\S+)\s+([0-9.]+)$", stdout, flags=re.M)}+++def test_cli_score_within_publishes_the_documented_metric(monkeypatch, tmp_path, capsys):+ from laya import evals_cli++ _patch_router(monkeypatch)+ dataset = _write_dataset(tmp_path, RUN_ROWS)+ assert evals_cli.main(["run", dataset, "--score-within", "0.25"]) == 0+ overall = _overall(capsys.readouterr().out)+ assert overall["score_within_0.25"] == pytest.approx(0.5), "one of the two score rows is inside"+ for name in ("choice_accuracy", "noul_accuracy", "score_mae", "mean_confidence", "ece"):+ assert name in overall, "--score-within adds to the defaults, it does not replace them"+++def test_cli_score_within_is_repeatable_per_column(monkeypatch, tmp_path, capsys):+ from laya import evals_cli++ _patch_router(monkeypatch)+ dataset = _write_dataset(tmp_path, RUN_ROWS)+ assert evals_cli.main(["run", dataset, "--score-within", "0.25", "--score-within", "0.5"]) == 0+ overall = _overall(capsys.readouterr().out)+ assert overall["score_within_0.25"] == pytest.approx(0.5)+ assert overall["score_within_0.5"] == pytest.approx(1.0), "the far row is inside 0.5"+++def test_cli_score_within_gate_decides_on_the_number(monkeypatch, tmp_path, capsys):+ from laya import evals_cli++ _patch_router(monkeypatch)+ dataset = _write_dataset(tmp_path, RUN_ROWS)+ assert evals_cli.main(["run", dataset, "--score-within", "0.25",+ "--min", "score_within_0.25=0.5"]) == 0+ capsys.readouterr()+ assert evals_cli.main(["run", dataset, "--score-within", "0.25",+ "--min", "score_within_0.25=0.6"]) == 1+ assert "below the minimum" in capsys.readouterr().err+++def test_cli_score_within_without_score_rows_says_so(monkeypatch, tmp_path, capsys):+ from laya import evals_cli++ _patch_router(monkeypatch)+ rows = [row for row in RUN_ROWS if row["state"] in ("intent", "flag")]+ dataset = _write_dataset(tmp_path, rows)+ assert evals_cli.main(["run", dataset, "--score-within", "0.25"]) == 0+ captured = capsys.readouterr()+ assert "score_within_0.25" not in captured.out, "no value is invented for it"+ assert "score_within_0.25 has no value" in captured.err+ assert "0 of 2 answered case(s) are score answers" in captured.err+++def test_cli_rejects_an_unusable_tolerance_before_loading_a_checkpoint(monkeypatch, tmp_path, capsys):+ from laya import evals_cli++ built = _patch_router(monkeypatch)+ dataset = _write_dataset(tmp_path, RUN_ROWS)+ for bad in ("0.25", "-0.1", "nan", "inf"):+ capsys.readouterr()+ code = evals_cli.main(["run", dataset, "--score-within", bad])+ if bad == "0.25":+ assert code == 0 and built == [None]+ continue+ assert code == 1, "%r would name a metric that is always 1.0 or always 0.0" % bad+ assert "--score-within" in capsys.readouterr().err+ assert built == [None], "the flag is rejected before a checkpoint is loaded"+++def test_cli_records_the_requested_tolerances(monkeypatch, tmp_path):+ from laya import evals_cli++ _patch_router(monkeypatch)+ dataset = _write_dataset(tmp_path, RUN_ROWS)+ out = tmp_path / "report.json"+ assert evals_cli.main(["run", dataset, "--score-within", "0.25", "--score-within", "0.5",+ "--json", str(out)]) == 0+ report = json.loads(out.read_text(encoding="utf-8"))+ assert report["config"]["score_within"] == [0.25, 0.5]+ assert "score_within_0.5" in report["overall"]+++def test_docs_and_the_cli_name_the_same_flags():+ import argparse+ from pathlib import Path++ from laya import evals_cli++ page = (Path(__file__).resolve().parent.parent / "docs" / "evals.md").read_text(encoding="utf-8")+ registered: set = set()+ for action in evals_cli._build_parser()._actions:+ if isinstance(action, argparse._SubParsersAction):+ for sub in action.choices.values():+ registered.update(sub._option_string_actions)+ quickstart = page.split("```bash", 1)[1].split("```", 1)[0]+ taught = set(re.findall(r"(?<![\w-])(--[a-z][a-z-]*)", quickstart))+ assert taught <= registered, "the quickstart teaches %s" % sorted(taught - registered)+ assert "--score-within" in taught, "the tolerance metric has to be reachable from the quickstart"+ metrics = page.split("## Metrics", 1)[1].split("\n## ", 1)[0]+ assert "score_within" in metrics and "--score-within" in metrics, \+ "the page that publishes the metric has to carry the flag that reaches it"