Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): evidence names the artifact a read failed on

Aashishacb56ab5025b
diff --git a/laya/evidence.py b/laya/evidence.py
index 9f2400eb..69692a63 100644
--- a/laya/evidence.py
+++ b/laya/evidence.py
@@ -169,21 +169,36 @@ def relate(config: Dict[str, Any], report: Dict[str, Any]) -> Dict[str, Any]:
return _entry(UNKNOWN, "no immutable shared checkpoint identity between this checkpoint artifact and the report")
+def _read_json_object(path: str) -> Dict[str, Any]:
+ """One artifact's JSON object, or a failure that names the file it came from.
+
+ A `json.JSONDecodeError` says what is wrong with the text and nothing about which file, and a
+ `UnicodeDecodeError` says the same about the encoding. `inspect_checkpoint` reads two of them
+ from two different arguments, so a refusal that omits the path cannot say which artifact is
+ broken -- the only thing the caller can act on. Same reason `laya/evals.py:155` prefixes its
+ dataset rows with the path.
+ """
+ try:
+ with open(path, "r", encoding="utf-8") as handle:
+ document = json.load(handle)
+ except json.JSONDecodeError as exc:
+ raise ValueError("%s is not valid JSON: %s" % (path, exc)) from exc
+ except UnicodeDecodeError as exc:
+ raise ValueError("%s is not UTF-8 text: %s" % (path, exc)) from exc
+ if not isinstance(document, dict):
+ raise ValueError("%s is not a JSON object" % path)
+ return document
+
+
def inspect_checkpoint(checkpoint_dir: str, report_path: Optional[str] = None) -> Dict[str, Any]:
"""Full read-only evidence inspection for one checkpoint (optionally one report)."""
config_path = os.path.join(checkpoint_dir, "rl_agent_config.json")
if not os.path.isdir(checkpoint_dir) or not os.path.isfile(config_path):
raise FileNotFoundError("no rl_agent_config.json under %r" % checkpoint_dir)
- with open(config_path, "r", encoding="utf-8") as handle:
- config = json.load(handle)
- if not isinstance(config, dict):
- raise ValueError("%s is not a JSON object" % config_path)
+ config = _read_json_object(config_path)
out: Dict[str, Any] = {"checkpoint": checkpoint_dir, "calibration": inspect_calibration(config)}
if report_path is not None:
- with open(report_path, "r", encoding="utf-8") as handle:
- report = json.load(handle)
- if not isinstance(report, dict):
- raise ValueError("%s is not a JSON object" % report_path)
+ report = _read_json_object(report_path)
out["report"] = inspect_report(report)
out["relationship"] = relate(config, report)
return out
diff --git a/tests/test_evidence.py b/tests/test_evidence.py
index f2841bfe..e6ec3aab 100644
--- a/tests/test_evidence.py
+++ b/tests/test_evidence.py
@@ -257,6 +257,50 @@ check("evidence CLI returns 0 on realistic pair", rc, 0)
rc_bad = evals_cli.main(["evidence", "--checkpoint", os.path.join(tmp, "nope")])
check("missing checkpoint dir -> usage error 2", rc_bad, 2)
+# ... but only that one shape of an unreadable path. `--report` pointing at a directory raised
+# `IsADirectoryError`, an `OSError` that is not a `FileNotFoundError`, so it walked past this
+# handler and out of `main`: a traceback and exit 1, the code `laya-evals` reserves for a quality
+# failure, on the run before it printed anything. The other rows were already usage errors and stay
+# so -- they are here because widening the catch to `OSError` must not swallow them, and because a
+# decode failure named no path, which matters most when the command takes two of them.
+import contextlib
+import io
+
+_dir_report = os.path.join(tmp, "a-directory")
+os.makedirs(_dir_report)
+_not_object = os.path.join(tmp, "not-an-object.json")
+with open(_not_object, "w", encoding="utf-8") as f:
+ json.dump([1, 2], f)
+_not_json = os.path.join(tmp, "not-json.json")
+with open(_not_json, "w", encoding="utf-8") as f:
+ f.write("{nope")
+_not_text = os.path.join(tmp, "not-text.json")
+with open(_not_text, "wb") as f:
+ f.write(b"\xff\xfe\x00\x01binary")
+_bad_config_ckpt = os.path.join(tmp, "ckpt-bad-config")
+os.makedirs(_bad_config_ckpt)
+with open(os.path.join(_bad_config_ckpt, "rl_agent_config.json"), "w", encoding="utf-8") as f:
+ f.write("{nope")
+
+for _label, _argv, _path in (
+ ("--report is a directory", ["evidence", "--checkpoint", ckpt, "--report", _dir_report],
+ _dir_report),
+ ("--report is JSON but not an object",
+ ["evidence", "--checkpoint", ckpt, "--report", _not_object], _not_object),
+ ("--report is not JSON", ["evidence", "--checkpoint", ckpt, "--report", _not_json],
+ _not_json),
+ ("--report is not UTF-8 text", ["evidence", "--checkpoint", ckpt, "--report", _not_text],
+ _not_text),
+ ("the checkpoint's own config is not JSON",
+ ["evidence", "--checkpoint", _bad_config_ckpt],
+ os.path.join(_bad_config_ckpt, "rl_agent_config.json"))):
+ _err = io.StringIO()
+ with contextlib.redirect_stderr(_err):
+ _rc = evals_cli.main(_argv)
+ check("evidence CLI: %s -> usage error 2" % _label, _rc, 2)
+ check_true("evidence CLI: %s prints no traceback" % _label, "Traceback" not in _err.getvalue())
+ check_true("evidence CLI: %s names the path" % _label, _path in _err.getvalue())
+
# ... and on the artifact the research harness actually writes: identity at the document's top
# level, no `config` key. Read through `inspect_checkpoint`/`format_summary`, which is the whole of
# what `evidence` prints, so this covers the summary's own shape as well as the verdicts.