Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(evals): let laya-evals run ask for a tolerance metric

Aashishf8a485583080
diff --git a/docs/evals.md b/docs/evals.md
index 3660ec3a..0503245a 100644
--- a/docs/evals.md
+++ b/docs/evals.md
@@ -15,7 +15,7 @@ laya-evals validate research/evals/fixture.jsonl
# score a labelled set on one checkpoint, with thresholds and a baseline
laya-evals run data.jsonl --model english --device cpu \
- --min-accuracy 0.8 --max-ece 0.05 --slice language \
+ --min-accuracy 0.8 --max-ece 0.05 --score-within 0.25 --slice language \
--json report.json --markdown report.md
# compare a saved report to a baseline
@@ -59,7 +59,14 @@ Each metric is computed per answer where it applies and aggregated over the data
| `latency_p50_ms`, `latency_p95_ms` | per request | wall time, informational |
Add `ScoreWithin(0.25)` to the evaluator list for a tolerance metric; the default set is
-`choice_accuracy`, `noul_accuracy`, `score_mae`, `mean_confidence`, plus `ece`.
+`choice_accuracy`, `noul_accuracy`, `score_mae`, `mean_confidence`, plus `ece`. From the CLI the
+same thing is one flag: `laya-evals run data.jsonl --score-within 0.25` reports `score_within_0.25`
+beside the defaults, and the flag repeats, so `--score-within 0.25 --score-within 0.5` reports both.
+
+A tolerance metric needs a `score` answer with a numeric label, so on a dataset without one it has
+no value: `run` names the metric it could not compute instead of publishing a silent zero, and a
+`--min` / `--max` gate naming that metric fails as missing. The tolerances a run was asked for are
+recorded in the report's `config` block, so a reviewed baseline says which columns it expects.
## Slices
diff --git a/laya/evals_cli.py b/laya/evals_cli.py
index e656eba6..ee5a6fb8 100644
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@ -5,6 +5,7 @@ Exit codes: 0 on success, 1 when a threshold or a baseline tolerance fails, 2 on
laya-evals validate research/evals/fixture.jsonl
laya-evals run research/evals/fixture.jsonl --model english --min-accuracy 0.8 --max-ece 0.1
laya-evals run data.jsonl --baseline baseline.json --tolerance choice_accuracy=0.02 --json out.json
+ laya-evals run data.jsonl --score-within 0.25 --min score_within_0.25=0.9
`laya eval ...` dispatches here from the main CLI, so both spellings work.
"""
@@ -12,6 +13,7 @@ from __future__ import annotations
import argparse
import json
+import math
import sys
from typing import Any, Dict, List, Optional, Sequence
@@ -47,6 +49,20 @@ def _parse_pairs(pairs: Optional[Sequence[str]]) -> Dict[str, float]:
return out
+def _score_within(tolerances: Optional[Sequence[float]]) -> List[evals.Evaluator]:
+ """Build the `ScoreWithin` evaluators `--score-within` asks for, in the order given.
+
+ A non-finite or negative tolerance would name a metric that is always 1.0 or always 0.0, and a
+ `--min` gate on that metric would then decide nothing, so the CLI rejects them up front.
+ """
+ out: List[evals.Evaluator] = []
+ for tolerance in tolerances or []:
+ if not math.isfinite(tolerance) or tolerance < 0.0:
+ raise EvalError("--score-within needs a finite tolerance >= 0, got %g" % tolerance)
+ out.append(evals.ScoreWithin(tolerance))
+ return out
+
+
def _build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="laya-evals",
description="Evaluate a Laya checkpoint on a labelled dataset.")
@@ -67,6 +83,10 @@ def _build_parser() -> argparse.ArgumentParser:
run.add_argument("--min-accuracy", type=float,
help="minimum accuracy (choice, else noul) for the whole dataset")
run.add_argument("--max-ece", type=float, help="maximum expected calibration error")
+ run.add_argument("--score-within", dest="score_within", type=float, action="append",
+ metavar="TOL",
+ help="also report score_within_TOL, the fraction of score answers within TOL "
+ "of the label; repeatable, and added to the default metrics")
run.add_argument("--min", action="append", metavar="METRIC=VALUE", help="minimum for any metric")
run.add_argument("--max", action="append", metavar="METRIC=VALUE", help="maximum for any metric")
run.add_argument("--slice", action="append", choices=("language", "model", "qid", "tag"),
@@ -115,6 +135,22 @@ def _cmd_validate(args) -> int:
return 0
+def _warn_no_value(extra: Sequence[evals.Evaluator], report: evals.EvalReport) -> None:
+ """Say so when a metric the caller asked for scored nothing.
+
+ `evaluate` drops a metric no answer applied to, so a requested `--score-within` over a
+ choice-only dataset would otherwise change the command and not the report.
+ """
+ missing = [evaluator.name for evaluator in extra if evaluator.name not in report.overall]
+ if not missing:
+ return
+ score_cases = sum(1 for case in report.cases if (case.get("answer") or {}).get("type") == "score")
+ for name in missing:
+ print("laya-evals: %s has no value: %d of %d answered case(s) are score answers, and the "
+ "metric needs one whose label is a number. A threshold naming it reports it missing."
+ % (name, score_cases, len(report.cases)), file=sys.stderr)
+
+
def _cmd_run(args) -> int:
import laya
@@ -122,10 +158,16 @@ def _cmd_run(args) -> int:
if args.model:
for example in dataset.examples: # --model is authoritative over per-row model
example.model = args.model
+ extra = _score_within(args.score_within) # before the checkpoint loads: a bad flag is cheap
runner: Any = RouterRunner(laya.Router(device=args.device, preload=False))
config = {"dataset": args.dataset, "model": args.model, "device": args.device}
- report = evals.evaluate(runner, dataset, batch_size=args.batch_size,
- on_error=args.on_error, config=config)
+ if extra:
+ config["score_within"] = [evaluator.tolerance for evaluator in extra]
+ report = evals.evaluate(runner, dataset, evaluators=evals.default_evaluators() + extra,
+ batch_size=args.batch_size, on_error=args.on_error, config=config)
+
+ if extra:
+ _warn_no_value(extra, report)
mins = _parse_pairs(args.min)
maxs = _parse_pairs(args.max)
diff --git a/tests/test_evals.py b/tests/test_evals.py
index 0e9ff2d7..8f7d50b4 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -3,6 +3,7 @@
Run: python -m pytest tests/test_evals.py -q
"""
import json
+import re
import pytest
@@ -36,6 +37,10 @@ def noul_answer(prob):
return {"type": "noul", "noul": prob, "confidence": max(prob, 1 - prob)}
+def score_answer(value, confidence=0.8):
+ return {"type": "score", "score": value, "confidence": confidence}
+
+
class StubRunner:
"""Returns fixed answers per state, so evaluation is deterministic and weight-free."""
@@ -229,3 +234,145 @@ def test_cli_rejects_a_malformed_tolerance():
with pytest.raises(EvalError):
evals_cli._parse_pairs(["choice_accuracy"])
+
+
+# ------------------------------------------------------------------ CLI run
+# One score row inside 0.25 of its label, one 0.3 away (inside 0.5 but not 0.25), plus a choice and
+# an noul row so every default metric is in the report too.
+RUN_ROWS = [
+ {"state": "near", "questions": QSCORE, "expected": {"quality": 4}},
+ {"state": "far", "questions": QSCORE, "expected": {"quality": 4}},
+ {"state": "intent", "questions": Q, "expected": {"intent": "a"}},
+ {"state": "flag", "questions": QNOUL, "expected": {"flag": True}},
+]
+RUN_ANSWERS = {
+ "near": {"quality": score_answer(4.0)},
+ "far": {"quality": score_answer(3.7)},
+ "intent": {"intent": choice_answer("a")},
+ "flag": {"flag": noul_answer(0.9)},
+}
+
+
+def _patch_router(monkeypatch, answers=None):
+ """Replace the checkpoint-loading `Router`, so `laya-evals run` needs no weights.
+
+ Returns the list the stand-in appends to when it is constructed, so a test can show a bad flag
+ fails before anything loads.
+ """
+ import laya
+
+ built: list = []
+
+ class FakeRouter:
+ def __init__(self, device=None, preload=False):
+ built.append(device)
+
+ def predict(self, state, questions, model=None):
+ return {"model": model or "stub", "answers": (answers or RUN_ANSWERS)[state]}
+
+ monkeypatch.setattr(laya, "Router", FakeRouter)
+ return built
+
+
+def _overall(stdout):
+ return {name: float(value) for name, value in
+ re.findall(r"^(\S+)\s+([0-9.]+)$", stdout, flags=re.M)}
+
+
+def test_cli_score_within_publishes_the_documented_metric(monkeypatch, tmp_path, capsys):
+ from laya import evals_cli
+
+ _patch_router(monkeypatch)
+ dataset = _write_dataset(tmp_path, RUN_ROWS)
+ assert evals_cli.main(["run", dataset, "--score-within", "0.25"]) == 0
+ overall = _overall(capsys.readouterr().out)
+ assert overall["score_within_0.25"] == pytest.approx(0.5), "one of the two score rows is inside"
+ for name in ("choice_accuracy", "noul_accuracy", "score_mae", "mean_confidence", "ece"):
+ assert name in overall, "--score-within adds to the defaults, it does not replace them"
+
+
+def test_cli_score_within_is_repeatable_per_column(monkeypatch, tmp_path, capsys):
+ from laya import evals_cli
+
+ _patch_router(monkeypatch)
+ dataset = _write_dataset(tmp_path, RUN_ROWS)
+ assert evals_cli.main(["run", dataset, "--score-within", "0.25", "--score-within", "0.5"]) == 0
+ overall = _overall(capsys.readouterr().out)
+ assert overall["score_within_0.25"] == pytest.approx(0.5)
+ assert overall["score_within_0.5"] == pytest.approx(1.0), "the far row is inside 0.5"
+
+
+def test_cli_score_within_gate_decides_on_the_number(monkeypatch, tmp_path, capsys):
+ from laya import evals_cli
+
+ _patch_router(monkeypatch)
+ dataset = _write_dataset(tmp_path, RUN_ROWS)
+ assert evals_cli.main(["run", dataset, "--score-within", "0.25",
+ "--min", "score_within_0.25=0.5"]) == 0
+ capsys.readouterr()
+ assert evals_cli.main(["run", dataset, "--score-within", "0.25",
+ "--min", "score_within_0.25=0.6"]) == 1
+ assert "below the minimum" in capsys.readouterr().err
+
+
+def test_cli_score_within_without_score_rows_says_so(monkeypatch, tmp_path, capsys):
+ from laya import evals_cli
+
+ _patch_router(monkeypatch)
+ rows = [row for row in RUN_ROWS if row["state"] in ("intent", "flag")]
+ dataset = _write_dataset(tmp_path, rows)
+ assert evals_cli.main(["run", dataset, "--score-within", "0.25"]) == 0
+ captured = capsys.readouterr()
+ assert "score_within_0.25" not in captured.out, "no value is invented for it"
+ assert "score_within_0.25 has no value" in captured.err
+ assert "0 of 2 answered case(s) are score answers" in captured.err
+
+
+def test_cli_rejects_an_unusable_tolerance_before_loading_a_checkpoint(monkeypatch, tmp_path, capsys):
+ from laya import evals_cli
+
+ built = _patch_router(monkeypatch)
+ dataset = _write_dataset(tmp_path, RUN_ROWS)
+ for bad in ("0.25", "-0.1", "nan", "inf"):
+ capsys.readouterr()
+ code = evals_cli.main(["run", dataset, "--score-within", bad])
+ if bad == "0.25":
+ assert code == 0 and built == [None]
+ continue
+ assert code == 1, "%r would name a metric that is always 1.0 or always 0.0" % bad
+ assert "--score-within" in capsys.readouterr().err
+ assert built == [None], "the flag is rejected before a checkpoint is loaded"
+
+
+def test_cli_records_the_requested_tolerances(monkeypatch, tmp_path):
+ from laya import evals_cli
+
+ _patch_router(monkeypatch)
+ dataset = _write_dataset(tmp_path, RUN_ROWS)
+ out = tmp_path / "report.json"
+ assert evals_cli.main(["run", dataset, "--score-within", "0.25", "--score-within", "0.5",
+ "--json", str(out)]) == 0
+ report = json.loads(out.read_text(encoding="utf-8"))
+ assert report["config"]["score_within"] == [0.25, 0.5]
+ assert "score_within_0.5" in report["overall"]
+
+
+def test_docs_and_the_cli_name_the_same_flags():
+ import argparse
+ from pathlib import Path
+
+ from laya import evals_cli
+
+ page = (Path(__file__).resolve().parent.parent / "docs" / "evals.md").read_text(encoding="utf-8")
+ registered: set = set()
+ for action in evals_cli._build_parser()._actions:
+ if isinstance(action, argparse._SubParsersAction):
+ for sub in action.choices.values():
+ registered.update(sub._option_string_actions)
+ quickstart = page.split("```bash", 1)[1].split("```", 1)[0]
+ taught = set(re.findall(r"(?<![\w-])(--[a-z][a-z-]*)", quickstart))
+ assert taught <= registered, "the quickstart teaches %s" % sorted(taught - registered)
+ assert "--score-within" in taught, "the tolerance metric has to be reachable from the quickstart"
+ metrics = page.split("## Metrics", 1)[1].split("\n## ", 1)[0]
+ assert "score_within" in metrics and "--score-within" in metrics, \
+ "the page that publishes the metric has to carry the flag that reaches it"