Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(evals): pin and report the checkpoint commit a run was scored on
Aashishd137c2b01377
diff --git a/docs/evals.md b/docs/evals.mdindex 3660ec3a..2d722be3 100644--- a/docs/evals.md+++ b/docs/evals.md@@ -88,7 +88,14 @@ Two CI surfaces use this:PR.The harness is deterministic for a fixed checkpoint revision, so a report is reproducible.-`run` records the dataset, model and device in the report's `config` block.+`run` records the dataset, model and device in the report's `config` block, plus `revisions`:+the commit each checkpoint that answered was actually loaded from. `--revision <SHA>` pins that+commit for every checkpoint the run loads, and `--revision english=<SHA>` pins one checkpoint+(repeatable) — which is the form an auto-routing run wants, since the three checkpoints are three+repositories and one commit cannot exist in all of them. Left unpinned, the run takes the+checkpoint's default branch and the report still says which commit answered, so a baseline drift+can be attributed to the weights or to the code. `laya/revisions.py` publishes reviewed commit+SHAs in `PINNED_REVISIONS` for callers who want to opt in.## Adding the real labelled setdiff --git a/laya/evals_cli.py b/laya/evals_cli.pyindex e656eba6..0d8ea284 100644--- a/laya/evals_cli.py+++ b/laya/evals_cli.py@@ -13,7 +13,7 @@ from __future__ import annotationsimport argparseimport jsonimport sys-from typing import Any, Dict, List, Optional, Sequence+from typing import Any, Dict, List, Optional, Sequence, Tuplefrom . import evalsfrom .evals import EvalError@@ -47,6 +47,33 @@ def _parse_pairs(pairs: Optional[Sequence[str]]) -> Dict[str, float]:return out+def _parse_revisions(pairs: Optional[Sequence[str]]) -> Tuple[Optional[str], Dict[str, str]]:+ """Read `--revision` into the two forms `Router` takes: one commit, or one per checkpoint.++ A bare value is the commit for every checkpoint this run loads; `NAME=SHA` pins one+ checkpoint, which is what an auto-routing run needs, because the three checkpoints are three+ Hub repositories and one commit cannot exist in all of them.+ """+ shared: Optional[str] = None+ per_model: Dict[str, str] = {}+ for pair in pairs or []:+ name, sep, value = pair.partition("=")+ if sep:+ name, value = name.strip(), value.strip()+ if not name or not value:+ raise EvalError("expected NAME=REVISION, got %r" % pair)+ per_model[name] = value+ else:+ value = pair.strip()+ if not value:+ raise EvalError("--revision wants a commit SHA or NAME=SHA, got %r" % pair)+ if shared is not None and shared != value:+ raise EvalError("--revision names two commits for every checkpoint: %r, %r"+ % (shared, value))+ shared = value+ return shared, per_model++def _build_parser() -> argparse.ArgumentParser:parser = argparse.ArgumentParser(prog="laya-evals",description="Evaluate a Laya checkpoint on a labelled dataset.")@@ -59,6 +86,11 @@ def _build_parser() -> argparse.ArgumentParser:run.add_argument("dataset")run.add_argument("--model", help="force a checkpoint instead of auto-routing")run.add_argument("--device", help="torch device, e.g. cpu or cuda")+ run.add_argument("--revision", action="append", metavar="SHA | NAME=SHA",+ help="pin the checkpoint commit: a bare SHA applies to every checkpoint this "+ "run loads, NAME=SHA pins one (repeatable). Unpinned runs fetch the "+ "checkpoint's default branch; the report records the commit that answered "+ "either way")run.add_argument("--batch-size", type=int, help="examples per forward pass when questions match")run.add_argument("--on-error", choices=("fail", "skip"), default="fail")run.add_argument("--baseline", help="a baseline report JSON to compare against")@@ -122,10 +154,25 @@ def _cmd_run(args) -> int:if args.model:for example in dataset.examples: # --model is authoritative over per-row modelexample.model = args.model- runner: Any = RouterRunner(laya.Router(device=args.device, preload=False))+ revision, revisions = _parse_revisions(args.revision)+ try:+ router = laya.Router(device=args.device, preload=False, revision=revision,+ revisions=revisions)+ except ValueError as exc:+ # `Router` already rejects a checkpoint name it does not know -- normalising case,+ # surrounding spaces and aliases on the way -- with the option list in the message. This+ # only turns that into `laya-evals: unknown model 'englishg'; choose one of …` instead of+ # a traceback, so a mistyped pin fails as a usage error and never starts a download.+ raise EvalError(str(exc)) from exc+ runner: Any = RouterRunner(router)config = {"dataset": args.dataset, "model": args.model, "device": args.device}report = evals.evaluate(runner, dataset, batch_size=args.batch_size,on_error=args.on_error, config=config)+ # Which commit answered belongs in the artifact a baseline is, and it can only be read after+ # the run: `preload=False` means no checkpoint is resident before the first row.+ # `loaded_revisions` reports the commit each resident agent came from -- the pin when there is+ # one, the commit the default branch resolved to when there is not, None for a local path.+ report.config = dict(report.config, revisions=router.loaded_revisions)mins = _parse_pairs(args.min)maxs = _parse_pairs(args.max)diff --git a/tests/test_evals.py b/tests/test_evals.pyindex 0e9ff2d7..68114d6a 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -229,3 +229,133 @@ def test_cli_rejects_a_malformed_tolerance():with pytest.raises(EvalError):evals_cli._parse_pairs(["choice_accuracy"])+++# --------------------------------------------------------------- --revision pinning+SHA = "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851"+++def _fake_router(monkeypatch, recorded):+ """Replace `laya.Router` with a weight-free stand-in that records how `_cmd_run` built it.++ It validates checkpoint names the way `Router.__init__` does, by calling the same+ `normalise_name`, so the fake refuses a typo for the real reason.+ """+ import laya++ class FakeRouter:+ def __init__(self, device=None, preload=None, revision=None, revisions=None):+ from laya.router import normalise_name+ recorded.update(device=device, preload=preload, revision=revision,+ revisions={normalise_name(k): v for k, v in (revisions or {}).items()})++ def predict(self, state, questions, model=None):+ return {"model": model or "english", "answers": {"intent": choice_answer("a")}}++ loaded_revisions = {"english": SHA}++ monkeypatch.setattr(laya, "Router", FakeRouter)+++def test_cli_forwards_the_pin_and_records_the_commit_that_answered(tmp_path, monkeypatch):+ from laya import evals_cli++ recorded = {}+ _fake_router(monkeypatch, recorded)+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])+ out = tmp_path / "report.json"+ assert evals_cli.main(["run", dataset, "--model", "english", "--device", "cpu",+ "--revision", "english=" + SHA, "--json", str(out)]) == 0+ assert recorded["revision"] is None+ assert recorded["revisions"] == {"english": SHA}+ # The report is the artifact a reviewer commits, so it has to carry the commit itself.+ assert json.loads(out.read_text())["config"]["revisions"] == {"english": SHA}+++def test_cli_bare_revision_pins_every_checkpoint(tmp_path, monkeypatch):+ from laya import evals_cli++ recorded = {}+ _fake_router(monkeypatch, recorded)+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])+ assert evals_cli.main(["run", dataset, "--revision", SHA, "--json",+ str(tmp_path / "r.json")]) == 0+ # A bare SHA is one commit for every checkpoint, so nothing is pinned per name.+ assert recorded["revision"] == SHA and recorded["revisions"] == {}+++def test_cli_records_an_unpinned_run_too(tmp_path, monkeypatch):+ """The default branch is what an unpinned baseline was taken on; the report must say so."""+ from laya import evals_cli++ recorded = {}+ _fake_router(monkeypatch, recorded)+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])+ out = tmp_path / "r.json"+ assert evals_cli.main(["run", dataset, "--json", str(out)]) == 0+ assert recorded["revision"] is None and recorded["revisions"] == {}+ assert json.loads(out.read_text())["config"]["revisions"] == {"english": SHA}+++def test_cli_rejects_a_typo_in_a_pinned_checkpoint_name(tmp_path, monkeypatch, capsys):+ from laya import evals_cli++ recorded = {}+ _fake_router(monkeypatch, recorded)+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])+ assert evals_cli.main(["run", dataset, "--revision", "englishg=" + SHA]) == 1+ err = capsys.readouterr().err+ assert "unknown model 'englishg'" in err+ assert "choose one of" in err, "the message comes from core, with the option list"+ assert "Traceback" not in err, "a mistyped pin is a usage error, not a crash"+ assert not recorded, "and it fails before any checkpoint is loaded"+++def test_cli_accepts_a_checkpoint_alias_in_a_pin(tmp_path, monkeypatch):+ """`Router` owns the alias table, so `en=` must reach it rather than be second-guessed here."""+ from laya import evals_cli++ recorded = {}+ _fake_router(monkeypatch, recorded)+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])+ assert evals_cli.main(["run", dataset, "--revision", "en=" + SHA]) == 0+ assert recorded["revisions"] == {"english": SHA}++[email protected]("pairs, expected", [+ (None, (None, {})),+ ([], (None, {})),+ ([SHA], (SHA, {})),+ ([SHA, SHA], (SHA, {})), # repeating one commit is agreement, not a clash+ (["en=" + SHA], (None, {"en": SHA})),+ (["en=" + SHA, SHA], (SHA, {"en": SHA})), # both forms at once: Router lets the pair win+])+def test_parse_revisions_accepts_both_forms(pairs, expected):+ from laya import evals_cli++ assert evals_cli._parse_revisions(pairs) == expected++[email protected]("pair, fragment", [+ ("=abc", "NAME=REVISION"),+ ("en=", "NAME=REVISION"),+ (" ", "commit SHA"),+ ("abc", None),+])+def test_parse_revisions_rejects_half_a_pair(pair, fragment):+ from laya import evals_cli++ if fragment is None:+ assert evals_cli._parse_revisions([pair]) == (pair, {})+ return+ with pytest.raises(EvalError) as exc:+ evals_cli._parse_revisions([pair])+ assert fragment in str(exc.value)+++def test_parse_revisions_rejects_two_different_bare_commits():+ from laya import evals_cli++ with pytest.raises(EvalError) as exc:+ evals_cli._parse_revisions(["abc", "def"])+ assert "two commits" in str(exc.value)