Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals-cli): fail a mistyped --model as a usage error, not a traceback

Aashish1b9702fb5b61
diff --git a/laya/evals_cli.py b/laya/evals_cli.py
index 0d8ff5ee..a9bf0fe8 100644
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@ -371,12 +371,19 @@ def _cmd_run(args) -> int:
if revisions:
pins["revisions"] = revisions
router = laya.Router(device=args.device, preload=False, **pins)
+ # `--model` is not a constructor argument: it is stamped onto every row and resolved by
+ # the first row itself. Resolving it here asks the Router that will answer, so the
+ # checkpoint names this run can serve are the names it accepts.
+ if args.model:
+ router.resolve(args.model)
except ValueError as exc:
# `Router` already rejects a checkpoint name it does not know -- normalising case,
- # surrounding spaces and aliases on the way -- with the option list in the message.
- # This only turns that into `laya-evals: unknown model 'englishg'; choose one of …`
- # instead of a traceback, so a mistyped pin fails as a usage error and never starts a
- # download.
+ # surrounding spaces and aliases on the way -- with the option list in the message, for
+ # a `--revision` pin at construction and for `--model` just above. This only turns that
+ # into `laya-evals: unknown model 'englishg'; choose one of …` instead of a traceback,
+ # so a mistyped checkpoint fails as a usage error and never starts a download. Left to
+ # the row, `--model` raised inside `evaluate`, where its ValueError escaped as exit 1 --
+ # the code that means a threshold or baseline failed.
raise EvalError(str(exc)) from exc
runner = RouterRunner(router)
config = {"dataset": args.dataset, "model": args.model, "device": args.device,
diff --git a/tests/test_evals.py b/tests/test_evals.py
index 24d58cec..696698bb 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -2345,8 +2345,9 @@ SHA = "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851"
def _fake_router(monkeypatch, recorded):
"""Replace `laya.Router` with a weight-free stand-in that records how `_cmd_run` built it.
- It validates checkpoint names the way `Router.__init__` does, by calling the same
- `normalise_name`, so the fake refuses a typo for the real reason.
+ It validates checkpoint names the way `Router.__init__` does and resolves a forced `--model` the
+ way `Router.resolve` does, both by calling the same `normalise_name`, so the fake refuses a typo
+ for the real reason.
"""
import laya
@@ -2356,6 +2357,13 @@ def _fake_router(monkeypatch, recorded):
recorded.update(device=device, preload=preload, revision=revision,
revisions={normalise_name(k): v for k, v in (revisions or {}).items()})
+ def resolve(self, name):
+ # `_cmd_run` resolves `--model` against the Router that answers, so the stand-in has to
+ # resolve too. It takes no `models=`, so its registry is the built-in table, and that is
+ # exactly what `normalise_name` checks.
+ from laya.router import normalise_name
+ return normalise_name(name)
+
def predict(self, state, questions, model=None, min_confidence=None):
# The gate is a real part of the Router interface since #361; a stub without it would
# turn a `--min-confidence` run into a TypeError and hide whether the CLI really
@@ -2434,6 +2442,50 @@ def test_cli_accepts_a_checkpoint_alias_in_a_pin(tmp_path, monkeypatch):
assert recorded["revisions"] == {"english": SHA}
+# ------------------------------------------------------------------ --model checkpoint name
+def test_cli_rejects_a_typo_in_the_forced_checkpoint_name(tmp_path, monkeypatch, capsys):
+ """`--model` is resolved by the first row, so a typo used to escape as a traceback and exit 1."""
+ from laya import evals_cli
+
+ recorded = {}
+ _fake_router(monkeypatch, recorded)
+ ran = []
+ real_evaluate = evals_cli.evals.evaluate
+
+ def spy(runner, ds, **kwargs):
+ ran.append(ds)
+ return real_evaluate(runner, ds, **kwargs)
+
+ monkeypatch.setattr(evals_cli.evals, "evaluate", spy)
+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])
+ assert evals_cli.main(["run", dataset, "--model", "englishg"]) == 2
+ err = capsys.readouterr().err
+ assert "unknown model 'englishg'" in err
+ assert "choose one of" in err, "the message comes from core, with the option list"
+ assert "Traceback" not in err, "a mistyped checkpoint is a usage error, not a crash"
+ assert not ran, "and it fails before a row is predicted"
+
+
+def test_cli_accepts_a_checkpoint_alias_for_the_forced_model(tmp_path, monkeypatch):
+ """Resolution is `Router.resolve`, which owns aliases and casing -- an exact-match check would fail here."""
+ from laya import evals_cli
+
+ recorded = {}
+ _fake_router(monkeypatch, recorded)
+ asked = {}
+ real_evaluate = evals_cli.evals.evaluate
+
+ def spy(runner, ds, **kwargs):
+ asked["model"] = ds.examples[0].model
+ return real_evaluate(runner, ds, **kwargs)
+
+ monkeypatch.setattr(evals_cli.evals, "evaluate", spy)
+ dataset = _write_dataset(tmp_path, [{"state": "s", "questions": Q, "expected": {"intent": "a"}}])
+ assert evals_cli.main(["run", dataset, "--model", "EN "]) == 0
+ # Validated, not rewritten: the spelling asked for is what the row and the report carry.
+ assert asked["model"] == "EN "
+
+
# --------------------------------------------------------------- opt-in slice gates
def _slice_gate_report(en_correct, zh_correct):
cases = [{"language": language, "scores": {"choice_accuracy": float(i < correct)},