Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(evals): pin and report the checkpoint commit a run was scored on

Aashishd137c2b01377
diff --git a/docs/evals.md b/docs/evals.md
index 3660ec3a..2d722be3 100644
--- a/docs/evals.md
+++ b/docs/evals.md
@@ -88,7 +88,14 @@ Two CI surfaces use this:
PR.
The harness is deterministic for a fixed checkpoint revision, so a report is reproducible.
-`run` records the dataset, model and device in the report's `config` block.
+`run` records the dataset, model and device in the report's `config` block, plus `revisions`:
+the commit each checkpoint that answered was actually loaded from. `--revision <SHA>` pins that
+commit for every checkpoint the run loads, and `--revision english=<SHA>` pins one checkpoint
+(repeatable) — which is the form an auto-routing run wants, since the three checkpoints are three
+repositories and one commit cannot exist in all of them. Left unpinned, the run takes the
+checkpoint's default branch and the report still says which commit answered, so a baseline drift
+can be attributed to the weights or to the code. `laya/revisions.py` publishes reviewed commit
+SHAs in `PINNED_REVISIONS` for callers who want to opt in.
## Adding the real labelled set
diff --git a/laya/evals_cli.py b/laya/evals_cli.py
index e656eba6..0d8ea284 100644
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@ -13,7 +13,7 @@ from __future__ import annotations
import argparse
import json
import sys
-from typing import Any, Dict, List, Optional, Sequence
+from typing import Any, Dict, List, Optional, Sequence, Tuple
from . import evals
from .evals import EvalError
@@ -47,6 +47,33 @@ def _parse_pairs(pairs: Optional[Sequence[str]]) -> Dict[str, float]:
return out
+def _parse_revisions(pairs: Optional[Sequence[str]]) -> Tuple[Optional[str], Dict[str, str]]:
+ """Read `--revision` into the two forms `Router` takes: one commit, or one per checkpoint.
+
+ A bare value is the commit for every checkpoint this run loads; `NAME=SHA` pins one
+ checkpoint, which is what an auto-routing run needs, because the three checkpoints are three
+ Hub repositories and one commit cannot exist in all of them.
+ """
+ shared: Optional[str] = None
+ per_model: Dict[str, str] = {}
+ for pair in pairs or []:
+ name, sep, value = pair.partition("=")
+ if sep:
+ name, value = name.strip(), value.strip()
+ if not name or not value:
+ raise EvalError("expected NAME=REVISION, got %r" % pair)
+ per_model[name] = value
+ else:
+ value = pair.strip()
+ if not value:
+ raise EvalError("--revision wants a commit SHA or NAME=SHA, got %r" % pair)
+ if shared is not None and shared != value:
+ raise EvalError("--revision names two commits for every checkpoint: %r, %r"
+ % (shared, value))
+ shared = value
+ return shared, per_model
+
+
def _build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="laya-evals",
description="Evaluate a Laya checkpoint on a labelled dataset.")
@@ -59,6 +86,11 @@ def _build_parser() -> argparse.ArgumentParser:
run.add_argument("dataset")
run.add_argument("--model", help="force a checkpoint instead of auto-routing")
run.add_argument("--device", help="torch device, e.g. cpu or cuda")
+ run.add_argument("--revision", action="append", metavar="SHA | NAME=SHA",
+ help="pin the checkpoint commit: a bare SHA applies to every checkpoint this "
+ "run loads, NAME=SHA pins one (repeatable). Unpinned runs fetch the "
+ "checkpoint's default branch; the report records the commit that answered "
+ "either way")
run.add_argument("--batch-size", type=int, help="examples per forward pass when questions match")
run.add_argument("--on-error", choices=("fail", "skip"), default="fail")
run.add_argument("--baseline", help="a baseline report JSON to compare against")
@@ -122,10 +154,25 @@ def _cmd_run(args) -> int:
if args.model:
for example in dataset.examples: # --model is authoritative over per-row model
example.model = args.model
- runner: Any = RouterRunner(laya.Router(device=args.device, preload=False))
+ revision, revisions = _parse_revisions(args.revision)
+ try:
+ router = laya.Router(device=args.device, preload=False, revision=revision,
+ revisions=revisions)
+ except ValueError as exc:
+ # `Router` already rejects a checkpoint name it does not know -- normalising case,
+ # surrounding spaces and aliases on the way -- with the option list in the message. This
+ # only turns that into `laya-evals: unknown model 'englishg'; choose one of …` instead of
+ # a traceback, so a mistyped pin fails as a usage error and never starts a download.
+ raise EvalError(str(exc)) from exc
+ runner: Any = RouterRunner(router)
config = {"dataset": args.dataset, "model": args.model, "device": args.device}
report = evals.evaluate(runner, dataset, batch_size=args.batch_size,
on_error=args.on_error, config=config)
+ # Which commit answered belongs in the artifact a baseline is, and it can only be read after
+ # the run: `preload=False` means no checkpoint is resident before the first row.
+ # `loaded_revisions` reports the commit each resident agent came from -- the pin when there is
+ # one, the commit the default branch resolved to when there is not, None for a local path.
+ report.config = dict(report.config, revisions=router.loaded_revisions)
mins = _parse_pairs(args.min)
maxs = _parse_pairs(args.max)
diff --git a/tests/test_evals.py b/tests/test_evals.py
index 0e9ff2d7..68114d6a 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -229,3 +229,133 @@ def test_cli_rejects_a_malformed_tolerance():
with pytest.raises(EvalError):
evals_cli._parse_pairs(["choice_accuracy"])
+
+
+# --------------------------------------------------------------- --revision pinning
+SHA = "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851"
+
+
+def _fake_router(monkeypatch, recorded):
+ """Replace `laya.Router` with a weight-free stand-in that records how `_cmd_run` built it.
+
+ It validates checkpoint names the way `Router.__init__` does, by calling the same
+ `normalise_name`, so the fake refuses a typo for the real reason.
+ """
+ import laya
+
+ class FakeRouter:
+ def __init__(self, device=None, preload=None, revision=None, revisions=None):
+ from laya.router import normalise_name
+ recorded.update(device=device, preload=preload, revision=revision,
+ revisions={normalise_name(k): v for k, v in (revisions or {}).items()})
+
+ def predict(self, state, questions, model=None):
+ return {"model": model or "english", "answers": {"intent": choice_answer("a")}}
+
+ loaded_revisions = {"english": SHA}
+
+ monkeypatch.setattr(laya, "Router", FakeRouter)
+
+
+def test_cli_forwards_the_pin_and_records_the_commit_that_answered(tmp_path, monkeypatch):
+ from laya import evals_cli
+
+ recorded = {}
+ _fake_router(monkeypatch, recorded)
+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])
+ out = tmp_path / "report.json"
+ assert evals_cli.main(["run", dataset, "--model", "english", "--device", "cpu",
+ "--revision", "english=" + SHA, "--json", str(out)]) == 0
+ assert recorded["revision"] is None
+ assert recorded["revisions"] == {"english": SHA}
+ # The report is the artifact a reviewer commits, so it has to carry the commit itself.
+ assert json.loads(out.read_text())["config"]["revisions"] == {"english": SHA}
+
+
+def test_cli_bare_revision_pins_every_checkpoint(tmp_path, monkeypatch):
+ from laya import evals_cli
+
+ recorded = {}
+ _fake_router(monkeypatch, recorded)
+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])
+ assert evals_cli.main(["run", dataset, "--revision", SHA, "--json",
+ str(tmp_path / "r.json")]) == 0
+ # A bare SHA is one commit for every checkpoint, so nothing is pinned per name.
+ assert recorded["revision"] == SHA and recorded["revisions"] == {}
+
+
+def test_cli_records_an_unpinned_run_too(tmp_path, monkeypatch):
+ """The default branch is what an unpinned baseline was taken on; the report must say so."""
+ from laya import evals_cli
+
+ recorded = {}
+ _fake_router(monkeypatch, recorded)
+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])
+ out = tmp_path / "r.json"
+ assert evals_cli.main(["run", dataset, "--json", str(out)]) == 0
+ assert recorded["revision"] is None and recorded["revisions"] == {}
+ assert json.loads(out.read_text())["config"]["revisions"] == {"english": SHA}
+
+
+def test_cli_rejects_a_typo_in_a_pinned_checkpoint_name(tmp_path, monkeypatch, capsys):
+ from laya import evals_cli
+
+ recorded = {}
+ _fake_router(monkeypatch, recorded)
+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])
+ assert evals_cli.main(["run", dataset, "--revision", "englishg=" + SHA]) == 1
+ err = capsys.readouterr().err
+ assert "unknown model 'englishg'" in err
+ assert "choose one of" in err, "the message comes from core, with the option list"
+ assert "Traceback" not in err, "a mistyped pin is a usage error, not a crash"
+ assert not recorded, "and it fails before any checkpoint is loaded"
+
+
+def test_cli_accepts_a_checkpoint_alias_in_a_pin(tmp_path, monkeypatch):
+ """`Router` owns the alias table, so `en=` must reach it rather than be second-guessed here."""
+ from laya import evals_cli
+
+ recorded = {}
+ _fake_router(monkeypatch, recorded)
+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])
+ assert evals_cli.main(["run", dataset, "--revision", "en=" + SHA]) == 0
+ assert recorded["revisions"] == {"english": SHA}
+
+
[email protected]("pairs, expected", [
+ (None, (None, {})),
+ ([], (None, {})),
+ ([SHA], (SHA, {})),
+ ([SHA, SHA], (SHA, {})), # repeating one commit is agreement, not a clash
+ (["en=" + SHA], (None, {"en": SHA})),
+ (["en=" + SHA, SHA], (SHA, {"en": SHA})), # both forms at once: Router lets the pair win
+])
+def test_parse_revisions_accepts_both_forms(pairs, expected):
+ from laya import evals_cli
+
+ assert evals_cli._parse_revisions(pairs) == expected
+
+
[email protected]("pair, fragment", [
+ ("=abc", "NAME=REVISION"),
+ ("en=", "NAME=REVISION"),
+ (" ", "commit SHA"),
+ ("abc", None),
+])
+def test_parse_revisions_rejects_half_a_pair(pair, fragment):
+ from laya import evals_cli
+
+ if fragment is None:
+ assert evals_cli._parse_revisions([pair]) == (pair, {})
+ return
+ with pytest.raises(EvalError) as exc:
+ evals_cli._parse_revisions([pair])
+ assert fragment in str(exc.value)
+
+
+def test_parse_revisions_rejects_two_different_bare_commits():
+ from laya import evals_cli
+
+ with pytest.raises(EvalError) as exc:
+ evals_cli._parse_revisions(["abc", "def"])
+ assert "two commits" in str(exc.value)