Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): a report laya-evals cannot read is a usage error, not exit 1

Aashish9df306a0123f
diff --git a/laya/evals_cli.py b/laya/evals_cli.py
index 55732f3c..723704f6 100644
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@ -257,8 +257,23 @@ def _build_parser() -> argparse.ArgumentParser:
def _load_report(path: str) -> Dict[str, Any]:
- with open(path, "r", encoding="utf-8") as handle:
- return json.load(handle)
+ # `main`'s usage-error handler promises that "an unreadable report" exits 2, and four report
+ # reads route through here. Only one shape of unreadable kept that promise: a missing path
+ # raised FileNotFoundError and landed in the handler, while a path that is a directory or on an
+ # unreadable volume, a file that is not valid JSON, and a file that is valid JSON but not an
+ # object each escaped as a traceback and exit 1 -- the code this CLI reserves for "a threshold
+ # or a baseline tolerance failed". A CI job cannot tell those apart, which is the one
+ # distinction the exit codes exist to draw. `laya-evals agreement` has always read its three
+ # runs this way; this makes the shared report loader satisfy the same contract.
+ try:
+ with open(path, "r", encoding="utf-8") as handle:
+ document = json.load(handle)
+ except (OSError, ValueError, RecursionError) as exc:
+ raise EvalError("cannot read report %r: %s" % (path, exc)) from exc
+ if not isinstance(document, dict):
+ raise EvalError("cannot read report %r: expected a JSON object, found %s"
+ % (path, type(document).__name__))
+ return document
def _check_thresholds(overall: Dict[str, float], mins: Dict[str, float],
@@ -489,7 +504,15 @@ def _cmd_run(args) -> int:
def _cmd_evidence(args) -> int:
try:
result = evidence.inspect_checkpoint(args.checkpoint, args.report)
- except (FileNotFoundError, ValueError) as exc:
+ except (OSError, ValueError) as exc:
+ # `FileNotFoundError` alone covered a missing path only. A `--report` that is a directory,
+ # or a checkpoint config the process may not read, raised an OSError subclass that walked
+ # out of `main` and left a traceback with exit 1 -- the code this CLI keeps for a quality
+ # failure -- next to the same mistake on a missing path, which exited 2. Both are caller
+ # mistakes about a named path, and the errno text names it. Nothing here reaches the Hub:
+ # `laya/evidence.py` imports `json` and `os` only, so there is no failed download to
+ # mistake for a usage error, which is why this handler is allowed to be wider than the
+ # `FileNotFoundError` one in `main`.
print("laya-evals: %s" % exc, file=sys.stderr)
return 2
print(evidence.format_summary(result))
diff --git a/tests/test_evals.py b/tests/test_evals.py
index cc469c2e..16702173 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -2154,6 +2154,80 @@ def test_cli_run_an_unwritable_output_path_is_a_usage_error(monkeypatch, tmp_pat
monkeypatch.setattr("builtins.open", real_open)
+def test_cli_report_reads_refuse_an_unreadable_report_with_code_2(monkeypatch, tmp_path, capsys):
+ """`main` promises an unreadable report is a usage error; only one shape of it kept that promise.
+
+ Every report read goes through `_load_report` -- `compare`'s report and baseline, and `run`'s
+ baseline -- and it opened bare. A path that does not exist raised `FileNotFoundError`, which
+ `main` narrows to exit 2, but the other ways a file fails to be a report each escaped: a path
+ that *is* a directory (`IsADirectoryError`), text that is not JSON (`JSONDecodeError`), JSON
+ that is valid but not an object (`AttributeError` one call later), and an errno that names no
+ path at all. Each left a traceback and exit 1 -- the code this CLI reserves for "a threshold or
+ a baseline tolerance failed" -- so a CI job could not tell a corrupt artifact from a regression.
+ The message has to name the path, because three of those four errors do not.
+ """
+ from laya import evals_cli
+
+ real_open = open # captured before any patch below delegates to it
+ document = json.dumps({"overall": {"choice_accuracy": 0.5}, "slices": {}, "cases": []})
+ good = tmp_path / "good.json"
+ good.write_text(document, encoding="utf-8")
+ not_json = tmp_path / "not-json.json"
+ not_json.write_text("{nope", encoding="utf-8")
+ not_object = tmp_path / "not-object.json"
+ not_object.write_text("[1, 2]", encoding="utf-8")
+ a_directory = tmp_path / "a-directory"
+ a_directory.mkdir()
+ denied = tmp_path / "denied.json"
+ denied.write_text(document, encoding="utf-8")
+
+ cases = [
+ ("a path that is a directory, not a file", str(a_directory), None),
+ ("a file that is not JSON", str(not_json), None),
+ ("a file that is valid JSON but not an object", str(not_object), None),
+ ("a volume that refuses the read", str(denied), OSError(13, "Permission denied")),
+ ]
+
+ def make_refuser(wanted, err):
+ """`open` for real, except that one path raises. A factory, so nothing binds late."""
+ def refuse(target, *a, **k):
+ if str(target) == wanted:
+ raise err
+ return real_open(target, *a, **k)
+ return refuse
+
+ for label, path, error in cases:
+ for position in ("report", "baseline"):
+ monkeypatch.setattr("builtins.open", real_open)
+ if error is not None:
+ monkeypatch.setattr("builtins.open", make_refuser(path, error))
+ argv = (["compare", path, "--baseline", str(good)] if position == "report" else
+ ["compare", str(good), "--baseline", path])
+ capsys.readouterr()
+ assert evals_cli.main(argv) == 2, "%s as %s: %s" % (label, position, argv)
+ err = capsys.readouterr().err
+ assert "Traceback" not in err, "%s as %s leaked a traceback: %s" % (label, position, err)
+ assert "cannot read report" in err, "%s as %s: %s" % (label, position, err)
+ assert path in err, "%s as %s did not name the path: %s" % (label, position, err)
+ if error is not None:
+ assert "Permission denied" in err, (
+ "%s as %s lost the errno the caller needs: %s" % (label, position, err))
+
+ # `run` reads the same function, on the same two shapes, after a whole evaluation pass.
+ _patch_router(monkeypatch)
+ dataset = _write_dataset(tmp_path, RUN_ROWS)
+ for label, path, error in cases[:2]:
+ monkeypatch.setattr("builtins.open", real_open)
+ if error is not None:
+ monkeypatch.setattr("builtins.open", make_refuser(path, error))
+ capsys.readouterr()
+ assert evals_cli.main(["run", dataset, "--baseline", path]) == 2, "%s under run" % label
+ err = capsys.readouterr().err
+ assert "Traceback" not in err, "%s under run leaked a traceback: %s" % (label, err)
+ assert "cannot read report" in err and path in err, "%s under run: %s" % (label, err)
+ monkeypatch.setattr("builtins.open", real_open)
+
+
def test_cli_run_refuses_a_baseline_from_a_different_dataset(monkeypatch, tmp_path, capsys):
"""The gate that a promotion decision rests on: two different runs cannot pass as one.