Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #593 from aashish254/feat-evals-cli-within

Nandakishor733a34635abe
diff --cc docs/evals.md
index 56af9c0e,0503245a..57bf788e
--- a/docs/evals.md
+++ b/docs/evals.md
@@@ -86,30 -56,18 +86,37 @@@ Each metric is computed per answer wher
| `score_within_<tol>` | `score` | fraction within an absolute tolerance |
| `ece` | any answer with a confidence | expected calibration error, 15 bins, computed on `answer["answer_confidence"]`, the calibrated probability Laya reports on every answer type |
| `mean_confidence` | any answer with a confidence | mean reported `answer["answer_confidence"]` |
-| `latency_p50_ms`, `latency_p95_ms` | per request | wall time, informational |
+| `latency_p50_ms`, `latency_p95_ms` | per request | wall time each request waited, informational -- see [batching](#batching-and-timing) |
+| `cost_per_decision_p50_ms`, `cost_per_decision_p95_ms` | per decision | a call's wall time divided by the rows it carried, informational |
Add `ScoreWithin(0.25)` to the evaluator list for a tolerance metric; the default set is
- `choice_accuracy`, `noul_accuracy`, `score_mae`, `mean_confidence`, plus `ece`.
+ `choice_accuracy`, `noul_accuracy`, `score_mae`, `mean_confidence`, plus `ece`. From the CLI the
+ same thing is one flag: `laya-evals run data.jsonl --score-within 0.25` reports `score_within_0.25`
+ beside the defaults, and the flag repeats, so `--score-within 0.25 --score-within 0.5` reports both.
+
+ A tolerance metric needs a `score` answer with a numeric label, so on a dataset without one it has
+ no value: `run` names the metric it could not compute instead of publishing a silent zero, and a
+ `--min` / `--max` gate naming that metric fails as missing. The tolerances a run was asked for are
+ recorded in the report's `config` block, so a reviewed baseline says which columns it expects.
+## Batching and timing
+
+`--batch-size N` scores up to N consecutive rows that share a checkpoint and a question schema in
+one call. Both timing metrics come from the same measurements and answer different questions:
+every row of a batch returns when the batch does, so its `latency` is the whole call, while its
+`cost_per_decision` is `1/N` of it. Batching therefore *raises* `latency_*` and *lowers*
+`cost_per_decision_*` on an unchanged set of decisions, and `--max latency_p50_ms=...` asks whether
+requests were served fast, not whether the run was cheap. With no `--batch-size` the two agree.
+
+`compare` ignores any `*_ms` metric unless a tolerance names it, so these never fail a baseline on
+timing noise. What the harness actually did -- the batch size asked for, the runner shape it
+resolved to, how many rows shared a call, and the largest chunk -- is recorded in the report's
+`config.timing`, because the flag alone does not say whether anything was batched. Those counters
+record the calls issued, not the calls that returned: with `on_error=skip`, a chunk whose call
+raised still counts in `rows_grouped` and `max_chunk`, next to its entries in `config.errored`. The
+two `*_ms` metrics count only the calls that returned, so a failed call never contributes a latency
+it did not measure.
+
## Slices
`compare` and `run` report overall numbers and, for `--slice language|model|qid|tag`, the same
diff --cc laya/evals_cli.py
index 4fef8794,ee5a6fb8..d2a3a8ff
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@@ -12,8 -13,9 +13,9 @@@ from __future__ import annotation
import argparse
import json
+ import math
import sys
-from typing import Any, Dict, List, Optional, Sequence
+from typing import Any, Dict, List, Optional, Sequence, Tuple
from . import evals
from .evals import EvalError
@@@ -78,33 -49,20 +80,47 @@@ def _parse_pairs(pairs: Optional[Sequen
return out
+def _parse_revisions(pairs: Optional[Sequence[str]]) -> Tuple[Optional[str], Dict[str, str]]:
+ """Read `--revision` into the two forms `Router` takes: one commit, or one per checkpoint.
+
+ A bare value is the commit for every checkpoint this run loads; `NAME=SHA` pins one
+ checkpoint, which is what an auto-routing run needs, because the three checkpoints are three
+ Hub repositories and one commit cannot exist in all of them.
+ """
+ shared: Optional[str] = None
+ per_model: Dict[str, str] = {}
+ for pair in pairs or []:
+ name, sep, value = pair.partition("=")
+ if sep:
+ name, value = name.strip(), value.strip()
+ if not name or not value:
+ raise EvalError("expected NAME=REVISION, got %r" % pair)
+ per_model[name] = value
+ else:
+ value = pair.strip()
+ if not value:
+ raise EvalError("--revision wants a commit SHA or NAME=SHA, got %r" % pair)
+ if shared is not None and shared != value:
+ raise EvalError("--revision names two commits for every checkpoint: %r, %r"
+ % (shared, value))
+ shared = value
+ return shared, per_model
+
+
+ def _score_within(tolerances: Optional[Sequence[float]]) -> List[evals.Evaluator]:
+ """Build the `ScoreWithin` evaluators `--score-within` asks for, in the order given.
+
+ A non-finite or negative tolerance would name a metric that is always 1.0 or always 0.0, and a
+ `--min` gate on that metric would then decide nothing, so the CLI rejects them up front.
+ """
+ out: List[evals.Evaluator] = []
+ for tolerance in tolerances or []:
+ if not math.isfinite(tolerance) or tolerance < 0.0:
+ raise EvalError("--score-within needs a finite tolerance >= 0, got %g" % tolerance)
+ out.append(evals.ScoreWithin(tolerance))
+ return out
+
+
def _build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="laya-evals",
description="Evaluate a Laya checkpoint on a labelled dataset.")
@@@ -191,47 -135,39 +211,73 @@@ def _cmd_validate(args) -> int
return 0
+ def _warn_no_value(extra: Sequence[evals.Evaluator], report: evals.EvalReport) -> None:
+ """Say so when a metric the caller asked for scored nothing.
+
+ `evaluate` drops a metric no answer applied to, so a requested `--score-within` over a
+ choice-only dataset would otherwise change the command and not the report.
+ """
+ missing = [evaluator.name for evaluator in extra if evaluator.name not in report.overall]
+ if not missing:
+ return
+ score_cases = sum(1 for case in report.cases if (case.get("answer") or {}).get("type") == "score")
+ for name in missing:
+ print("laya-evals: %s has no value: %d of %d answered case(s) are score answers, and the "
+ "metric needs one whose label is a number. A threshold naming it reports it missing."
+ % (name, score_cases, len(report.cases)), file=sys.stderr)
+
+
def _cmd_run(args) -> int:
- import laya
-
dataset = evals.Dataset.from_jsonl(args.dataset)
if args.model:
for example in dataset.examples: # --model is authoritative over per-row model
example.model = args.model
+ extra = _score_within(args.score_within) # before the checkpoint loads: a bad flag is cheap
- runner: Any = RouterRunner(laya.Router(device=args.device, preload=False))
+ revision, revisions = _parse_revisions(args.revision)
+ router = None
+ if args.onnx:
+ from .onnx_agent import ONNXAgent
+ # The export was produced from one checkpoint; --model names it (the ONNXAgent load
+ # needs its config and tokenizer), so the default is the english bundle repo. A bare
+ # --revision pins that checkpoint's config and tokenizer download.
+ if revisions:
+ raise EvalError("--revision NAME=SHA needs the Router; with --onnx pass one bare SHA")
+ agent = ONNXAgent(args.model or "convaiinnovations/laya", onnx_path=args.onnx,
+ revision=revision)
+ runner: Any = OnnxRunner(agent)
+ else:
+ import laya
+ try:
- router = laya.Router(device=args.device, preload=False, revision=revision,
- revisions=revisions)
++ pins: Dict[str, Any] = {}
++ if revision is not None:
++ pins["revision"] = revision
++ if revisions:
++ pins["revisions"] = revisions
++ router = laya.Router(device=args.device, preload=False, **pins)
+ except ValueError as exc:
+ # `Router` already rejects a checkpoint name it does not know -- normalising case,
+ # surrounding spaces and aliases on the way -- with the option list in the message.
+ # This only turns that into `laya-evals: unknown model 'englishg'; choose one of …`
+ # instead of a traceback, so a mistyped pin fails as a usage error and never starts a
+ # download.
+ raise EvalError(str(exc)) from exc
+ runner = RouterRunner(router)
config = {"dataset": args.dataset, "model": args.model, "device": args.device}
+ if args.onnx:
+ config["onnx"] = args.onnx
- report = evals.evaluate(runner, dataset, batch_size=args.batch_size,
- on_error=args.on_error, config=config)
+ if extra:
+ config["score_within"] = [evaluator.tolerance for evaluator in extra]
+ report = evals.evaluate(runner, dataset, evaluators=evals.default_evaluators() + extra,
+ batch_size=args.batch_size, on_error=args.on_error, config=config)
+ # Which commit answered belongs in the artifact a baseline is, and it can only be read after
+ # the run: `preload=False` means no checkpoint is resident before the first row.
+ # `loaded_revisions` reports the commit each resident agent came from -- the pin when there is
+ # one, the commit the default branch resolved to when there is not, None for a local path.
+ if router is not None:
- report.config = dict(report.config, revisions=router.loaded_revisions)
++ report.config = dict(report.config, revisions=getattr(router, "loaded_revisions", {}))
+
+ if extra:
+ _warn_no_value(extra, report)
mins = _parse_pairs(args.min)
maxs = _parse_pairs(args.max)
diff --cc tests/test_evals.py
index 17c020d8,f73f5580..ce0db32a
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@@ -630,131 -236,144 +635,276 @@@ def test_cli_rejects_a_malformed_tolera
evals_cli._parse_pairs(["choice_accuracy"])
+ # ------------------------------------------------------------------ CLI run
+ # One score row inside 0.25 of its label, one 0.3 away (inside 0.5 but not 0.25), plus a choice and
+ # an noul row so every default metric is in the report too.
+ RUN_ROWS = [
+ {"state": "near", "questions": QSCORE, "expected": {"quality": 4}},
+ {"state": "far", "questions": QSCORE, "expected": {"quality": 4}},
+ {"state": "intent", "questions": Q, "expected": {"intent": "a"}},
+ {"state": "flag", "questions": QNOUL, "expected": {"flag": True}},
+ ]
+ RUN_ANSWERS = {
+ "near": {"quality": score_answer(4.0)},
+ "far": {"quality": score_answer(3.7)},
+ "intent": {"intent": choice_answer("a")},
+ "flag": {"flag": noul_answer(0.9)},
+ }
+
+
+ def _patch_router(monkeypatch, answers=None):
+ """Replace the checkpoint-loading `Router`, so `laya-evals run` needs no weights.
+
+ Returns the list the stand-in appends to when it is constructed, so a test can show a bad flag
+ fails before anything loads.
+ """
+ import laya
+
+ built: list = []
+
+ class FakeRouter:
+ def __init__(self, device=None, preload=False):
+ built.append(device)
+
+ def predict(self, state, questions, model=None):
+ return {"model": model or "stub", "answers": (answers or RUN_ANSWERS)[state]}
+
+ monkeypatch.setattr(laya, "Router", FakeRouter)
+ return built
+
+
+ def _overall(stdout):
+ return {name: float(value) for name, value in
+ re.findall(r"^(\S+)\s+([0-9.]+)$", stdout, flags=re.M)}
+
+
+ def test_cli_score_within_publishes_the_documented_metric(monkeypatch, tmp_path, capsys):
+ from laya import evals_cli
+
+ _patch_router(monkeypatch)
+ dataset = _write_dataset(tmp_path, RUN_ROWS)
+ assert evals_cli.main(["run", dataset, "--score-within", "0.25"]) == 0
+ overall = _overall(capsys.readouterr().out)
+ assert overall["score_within_0.25"] == pytest.approx(0.5), "one of the two score rows is inside"
+ for name in ("choice_accuracy", "noul_accuracy", "score_mae", "mean_confidence", "ece"):
+ assert name in overall, "--score-within adds to the defaults, it does not replace them"
+
+
+ def test_cli_score_within_is_repeatable_per_column(monkeypatch, tmp_path, capsys):
+ from laya import evals_cli
+
+ _patch_router(monkeypatch)
+ dataset = _write_dataset(tmp_path, RUN_ROWS)
+ assert evals_cli.main(["run", dataset, "--score-within", "0.25", "--score-within", "0.5"]) == 0
+ overall = _overall(capsys.readouterr().out)
+ assert overall["score_within_0.25"] == pytest.approx(0.5)
+ assert overall["score_within_0.5"] == pytest.approx(1.0), "the far row is inside 0.5"
+
+
+ def test_cli_score_within_gate_decides_on_the_number(monkeypatch, tmp_path, capsys):
+ from laya import evals_cli
+
+ _patch_router(monkeypatch)
+ dataset = _write_dataset(tmp_path, RUN_ROWS)
+ assert evals_cli.main(["run", dataset, "--score-within", "0.25",
+ "--min", "score_within_0.25=0.5"]) == 0
+ capsys.readouterr()
+ assert evals_cli.main(["run", dataset, "--score-within", "0.25",
+ "--min", "score_within_0.25=0.6"]) == 1
+ assert "below the minimum" in capsys.readouterr().err
+
+
+ def test_cli_score_within_without_score_rows_says_so(monkeypatch, tmp_path, capsys):
+ from laya import evals_cli
+
+ _patch_router(monkeypatch)
+ rows = [row for row in RUN_ROWS if row["state"] in ("intent", "flag")]
+ dataset = _write_dataset(tmp_path, rows)
+ assert evals_cli.main(["run", dataset, "--score-within", "0.25"]) == 0
+ captured = capsys.readouterr()
+ assert "score_within_0.25" not in captured.out, "no value is invented for it"
+ assert "score_within_0.25 has no value" in captured.err
+ assert "0 of 2 answered case(s) are score answers" in captured.err
+
+
+ def test_cli_rejects_an_unusable_tolerance_before_loading_a_checkpoint(monkeypatch, tmp_path, capsys):
+ from laya import evals_cli
+
+ built = _patch_router(monkeypatch)
+ dataset = _write_dataset(tmp_path, RUN_ROWS)
+ for bad in ("0.25", "-0.1", "nan", "inf"):
+ capsys.readouterr()
+ code = evals_cli.main(["run", dataset, "--score-within", bad])
+ if bad == "0.25":
+ assert code == 0 and built == [None]
+ continue
+ assert code == 1, "%r would name a metric that is always 1.0 or always 0.0" % bad
+ assert "--score-within" in capsys.readouterr().err
+ assert built == [None], "the flag is rejected before a checkpoint is loaded"
+
+
+ def test_cli_records_the_requested_tolerances(monkeypatch, tmp_path):
+ from laya import evals_cli
+
+ _patch_router(monkeypatch)
+ dataset = _write_dataset(tmp_path, RUN_ROWS)
+ out = tmp_path / "report.json"
+ assert evals_cli.main(["run", dataset, "--score-within", "0.25", "--score-within", "0.5",
+ "--json", str(out)]) == 0
+ report = json.loads(out.read_text(encoding="utf-8"))
+ assert report["config"]["score_within"] == [0.25, 0.5]
+ assert "score_within_0.5" in report["overall"]
+
+
+ def test_docs_and_the_cli_name_the_same_flags():
+ import argparse
+ from pathlib import Path
+
+ from laya import evals_cli
+
+ page = (Path(__file__).resolve().parent.parent / "docs" / "evals.md").read_text(encoding="utf-8")
+ registered: set = set()
+ for action in evals_cli._build_parser()._actions:
+ if isinstance(action, argparse._SubParsersAction):
+ for sub in action.choices.values():
+ registered.update(sub._option_string_actions)
+ quickstart = page.split("```bash", 1)[1].split("```", 1)[0]
- taught = set(re.findall(r"(?<![\w-])(--[a-z][a-z-]*)", page))
++ # Lines that run another script (the ONNX exporter) teach that script's flags, not these.
++ evals_lines = "\n".join(line for line in page.splitlines() if "export_onnx.py" not in line)
++ taught = set(re.findall(r"(?<![\w-])(--[a-z][a-z-]*)", evals_lines))
+ assert taught <= registered, "the page teaches %s, which no subcommand registers" % sorted(
+ taught - registered)
+ assert "--score-within" in quickstart, "the tolerance metric has to be reachable from the quickstart"
+ metrics = page.split("## Metrics", 1)[1].split("\n## ", 1)[0]
+ assert "score_within" in metrics and "--score-within" in metrics, \
+ "the section that publishes the metric has to carry the flag that reaches it"
++
++
+# --------------------------------------------------------------- --revision pinning
+SHA = "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851"
+
+
+def _fake_router(monkeypatch, recorded):
+ """Replace `laya.Router` with a weight-free stand-in that records how `_cmd_run` built it.
+
+ It validates checkpoint names the way `Router.__init__` does, by calling the same
+ `normalise_name`, so the fake refuses a typo for the real reason.
+ """
+ import laya
+
+ class FakeRouter:
+ def __init__(self, device=None, preload=None, revision=None, revisions=None):
+ from laya.router import normalise_name
+ recorded.update(device=device, preload=preload, revision=revision,
+ revisions={normalise_name(k): v for k, v in (revisions or {}).items()})
+
+ def predict(self, state, questions, model=None):
+ return {"model": model or "english", "answers": {"intent": choice_answer("a")}}
+
+ loaded_revisions = {"english": SHA}
+
+ monkeypatch.setattr(laya, "Router", FakeRouter)
+
+
+def test_cli_forwards_the_pin_and_records_the_commit_that_answered(tmp_path, monkeypatch):
+ from laya import evals_cli
+
+ recorded = {}
+ _fake_router(monkeypatch, recorded)
+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])
+ out = tmp_path / "report.json"
+ assert evals_cli.main(["run", dataset, "--model", "english", "--device", "cpu",
+ "--revision", "english=" + SHA, "--json", str(out)]) == 0
+ assert recorded["revision"] is None
+ assert recorded["revisions"] == {"english": SHA}
+ # The report is the artifact a reviewer commits, so it has to carry the commit itself.
+ assert json.loads(out.read_text())["config"]["revisions"] == {"english": SHA}
+
+
+def test_cli_bare_revision_pins_every_checkpoint(tmp_path, monkeypatch):
+ from laya import evals_cli
+
+ recorded = {}
+ _fake_router(monkeypatch, recorded)
+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])
+ assert evals_cli.main(["run", dataset, "--revision", SHA, "--json",
+ str(tmp_path / "r.json")]) == 0
+ # A bare SHA is one commit for every checkpoint, so nothing is pinned per name.
+ assert recorded["revision"] == SHA and recorded["revisions"] == {}
+
+
+def test_cli_records_an_unpinned_run_too(tmp_path, monkeypatch):
+ """The default branch is what an unpinned baseline was taken on; the report must say so."""
+ from laya import evals_cli
+
+ recorded = {}
+ _fake_router(monkeypatch, recorded)
+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])
+ out = tmp_path / "r.json"
+ assert evals_cli.main(["run", dataset, "--json", str(out)]) == 0
+ assert recorded["revision"] is None and recorded["revisions"] == {}
+ assert json.loads(out.read_text())["config"]["revisions"] == {"english": SHA}
+
+
+def test_cli_rejects_a_typo_in_a_pinned_checkpoint_name(tmp_path, monkeypatch, capsys):
+ from laya import evals_cli
+
+ recorded = {}
+ _fake_router(monkeypatch, recorded)
+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])
+ assert evals_cli.main(["run", dataset, "--revision", "englishg=" + SHA]) == 1
+ err = capsys.readouterr().err
+ assert "unknown model 'englishg'" in err
+ assert "choose one of" in err, "the message comes from core, with the option list"
+ assert "Traceback" not in err, "a mistyped pin is a usage error, not a crash"
+ assert not recorded, "and it fails before any checkpoint is loaded"
+
+
+def test_cli_accepts_a_checkpoint_alias_in_a_pin(tmp_path, monkeypatch):
+ """`Router` owns the alias table, so `en=` must reach it rather than be second-guessed here."""
+ from laya import evals_cli
+
+ recorded = {}
+ _fake_router(monkeypatch, recorded)
+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])
+ assert evals_cli.main(["run", dataset, "--revision", "en=" + SHA]) == 0
+ assert recorded["revisions"] == {"english": SHA}
+
+
[email protected]("pairs, expected", [
+ (None, (None, {})),
+ ([], (None, {})),
+ ([SHA], (SHA, {})),
+ ([SHA, SHA], (SHA, {})), # repeating one commit is agreement, not a clash
+ (["en=" + SHA], (None, {"en": SHA})),
+ (["en=" + SHA, SHA], (SHA, {"en": SHA})), # both forms at once: Router lets the pair win
+])
+def test_parse_revisions_accepts_both_forms(pairs, expected):
+ from laya import evals_cli
+
+ assert evals_cli._parse_revisions(pairs) == expected
+
+
[email protected]("pair, fragment", [
+ ("=abc", "NAME=REVISION"),
+ ("en=", "NAME=REVISION"),
+ (" ", "commit SHA"),
+ ("abc", None),
+])
+def test_parse_revisions_rejects_half_a_pair(pair, fragment):
+ from laya import evals_cli
+
+ if fragment is None:
+ assert evals_cli._parse_revisions([pair]) == (pair, {})
+ return
+ with pytest.raises(EvalError) as exc:
+ evals_cli._parse_revisions([pair])
+ assert fragment in str(exc.value)
+
+
+def test_parse_revisions_rejects_two_different_bare_commits():
+ from laya import evals_cli
+
+ with pytest.raises(EvalError) as exc:
+ evals_cli._parse_revisions(["abc", "def"])
+ assert "two commits" in str(exc.value)