Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): refuse two commits for one --revision checkpoint

Aashish661d0404a2ec
diff --git a/docs/evals.md b/docs/evals.md
index d9be6c21..d74bd190 100644
--- a/docs/evals.md
+++ b/docs/evals.md
@@ -583,9 +583,13 @@ The harness is deterministic for a fixed checkpoint revision, so a report is rep
`run` records the dataset, model and device, plus the [timing](#batching-and-timing) facts of the
run, in the report's `config` block, and `revisions`: the commit each checkpoint that answered was
actually loaded from. `--revision <SHA>` pins that commit for every checkpoint the run loads, and
-`--revision english=<SHA>` pins one checkpoint (repeatable) — which is the form an auto-routing run
+`--revision english=<SHA>` pins one checkpoint — which is the form an auto-routing run
wants, since the three checkpoints are three repositories and one commit cannot exist in all of
-them. Left unpinned, the run takes the checkpoint's default branch and the report still says which
+them. Repeat a pin to say the same commit twice; two different commits for one checkpoint — however
+it is spelled, so `en=` beside `english=` counts — exit 2 instead of loading whichever came last,
+because `revisions` reports the commit that ANSWERED and a dropped pin would leave no trace. The
+same refusal covers two different bare commits.
+Left unpinned, the run takes the checkpoint's default branch and the report still says which
commit answered, so a baseline drift can be attributed to the weights or to the code.
`laya/revisions.py` publishes reviewed commit SHAs in `PINNED_REVISIONS` for callers who want to
opt in. With `--onnx`, only a bare `--revision <SHA>` applies, to the config and tokenizer download.
diff --git a/laya/evals_cli.py b/laya/evals_cli.py
index 9f261d83..de3210fe 100644
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@ -129,16 +129,35 @@ def _parse_revisions(pairs: Optional[Sequence[str]]) -> Tuple[Optional[str], Dic
A bare value is the commit for every checkpoint this run loads; `NAME=SHA` pins one
checkpoint, which is what an auto-routing run needs, because the three checkpoints are three
Hub repositories and one commit cannot exist in all of them.
+
+ Repeating a pin is agreement, not a clash, and is accepted either way. Two different commits for
+ one checkpoint -- every checkpoint, or the same one -- are refused rather than resolved to
+ whichever pair came last: a run loads one commit per checkpoint, and the report's `revisions`
+ block records the commit that ANSWERED, so the dropped pin would leave no trace anywhere a
+ reader could see the ambiguity in.
"""
+ # The target is a checkpoint, not a spelling: `Router.resolve` folds case and aliases onto the
+ # key it loads, so `en=` beside `english=` asks for two commits of one checkpoint. Asking the
+ # same function the Router asks keeps the CLI's grouping and the Router's from diverging.
+ from .router import canonical_name
+
shared: Optional[str] = None
per_model: Dict[str, str] = {}
+ pinned: Dict[str, Tuple[str, str]] = {} # checkpoint -> (spelling it arrived as, commit)
for pair in pairs or []:
name, sep, value = pair.partition("=")
if sep:
name, value = name.strip(), value.strip()
if not name or not value:
raise EvalError("expected NAME=REVISION, got %r" % pair)
+ checkpoint = canonical_name(name)
+ earlier = pinned.get(checkpoint)
+ if earlier is not None and earlier[1] != value:
+ spelled = "" if earlier[0] == name else " (given as %r and %r)" % (earlier[0], name)
+ raise EvalError("--revision names two commits for %r%s: %r, %r"
+ % (checkpoint, spelled, earlier[1], value))
per_model[name] = value
+ pinned[checkpoint] = (name, value)
else:
value = pair.strip()
if not value:
diff --git a/tests/test_evals.py b/tests/test_evals.py
index f8ad6564..6bc96c3c 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -3127,6 +3127,9 @@ def test_slice_gate_fails_closed_on_identity_or_skipped_cases(tmp_path):
([SHA, SHA], (SHA, {})), # repeating one commit is agreement, not a clash
(["en=" + SHA], (None, {"en": SHA})),
(["en=" + SHA, SHA], (SHA, {"en": SHA})), # both forms at once: Router lets the pair win
+ (["en=" + SHA, "en=" + SHA], (None, {"en": SHA})), # one pin twice is agreement
+ (["en=" + SHA, "fr=" + SHA], (None, {"en": SHA, "fr": SHA})), # two names are not a clash
+ (["en=" + SHA, "english=" + SHA], (None, {"en": SHA, "english": SHA})), # one commit, two spellings
])
def test_parse_revisions_accepts_both_forms(pairs, expected):
from laya import evals_cli
@@ -3151,6 +3154,35 @@ def test_parse_revisions_rejects_half_a_pair(pair, fragment):
assert fragment in str(exc.value)
[email protected]("pairs, spelled", [
+ (["english=abc", "english=def"], None),
+ (["English=abc", "english=def"], ("English", "english")), # case folds onto one checkpoint
+ (["en=abc", "english=def"], ("en", "english")), # `en` is an alias of `english`
+ (["laya=abc", "default=def"], ("laya", "default")), # two aliases, neither a real key
+])
+def test_parse_revisions_rejects_two_commits_for_one_checkpoint(pairs, spelled):
+ """The `NAME=SHA` form refuses the clash the bare form already refuses.
+
+ `--revision english=abc --revision english=def` named one checkpoint and two commits for it. The
+ run loads one commit, so one of them is dropped -- and the report's `revisions` block records the
+ commit that answered, never the request, so the survivor is the only thing left to read.
+ `Router.resolve` folds case and aliases onto the key it loads, so the same is true however the
+ checkpoint is spelled; the refusal follows the checkpoint, not the spelling.
+ """
+ from laya import evals_cli
+
+ with pytest.raises(EvalError) as exc:
+ evals_cli._parse_revisions(pairs)
+ message = str(exc.value)
+ assert "two commits" in message and "'english'" in message
+ # named in the order they were given, the way the bare-form message reports its two commits
+ assert message.index("'abc'") < message.index("'def'")
+ if spelled is None:
+ assert "given as" not in message
+ else:
+ assert "given as %r and %r" % spelled in message
+
+
def test_parse_revisions_rejects_two_different_bare_commits():
from laya import evals_cli