Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(evals): read-only evidence inspection over checkpoint and eval artifacts

Bruce-Yii473a35f335be
diff --git a/docs/evals.md b/docs/evals.md
index 8a65d0ec..9bcfc862 100644
--- a/docs/evals.md
+++ b/docs/evals.md
@@ -392,3 +392,25 @@ opt in. With `--onnx`, only a bare `--revision <SHA>` applies, to the config and
Drop a JSONL in `research/evals/` and a reviewed baseline beside it, then point a workflow (or
`research/evals/check_regression.py`) at both. The format is the same as the fixture; nothing in
the harness knows about MASSIVE.
+
+## Evidence inspection
+
+`evidence` reads — never writes — the artifacts a fine-tune/eval run already persists and says
+what evidence exists, what is missing, and what is insufficient. It loads no model weights and
+does not require torch.
+
+```bash
+laya-evals evidence --checkpoint ./my-checkpoint [--report report.json] [--json out.json]
+```
+
+A checkpoint directory must contain `rl_agent_config.json`. Calibration evidence is derived from
+the persisted `training.laya_train_calibration` block: a legacy checkpoint with no training
+metadata is `UNKNOWN`; a question type with zero items is `MISSING`; fewer than `MIN_TYPE_N` items
+or a fit with recorded limitations (below `CALIB_WARN_N`, clamped, unchanged at 1.0) is
+`INSUFFICIENT`; a clean supported fit is `PRESENT`. When torch is unavailable, the thresholds
+fall back to the documented `MIN_TYPE_N`/`CALIB_WARN_N` values and the summary says so.
+
+The eval report's identity fields (`schema`, `dataset_sha256`, `questions_sha256`, `laya_version`)
+are reported with the same semantics as `run`. The checkpoint↔report relationship is claimed only
+when the shared identity fields provably agree; a real conflict is `INCOMPARABLE`, and shared
+nothing is `UNKNOWN` — never an inference from paths or names.
diff --git a/laya/evals_cli.py b/laya/evals_cli.py
index 66897dc1..3bb9b4ea 100644
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@ -17,7 +17,7 @@ import math
import sys
from typing import Any, Dict, List, Optional, Sequence, Tuple
-from . import _eval_policy, evals
+from . import _eval_policy, evals, evidence
from .evals import EvalError
@@ -230,6 +230,14 @@ def _build_parser() -> argparse.ArgumentParser:
help="allowed absolute drift; repeatable")
compare.add_argument("--gate-policy", metavar="FILE",
help="apply opt-in per-slice quality rules from a JSON policy")
+ evidence_cmd = sub.add_parser("evidence",
+ help="read-only evidence inspection over a checkpoint config and an eval report")
+ evidence_cmd.add_argument("--checkpoint", required=True,
+ help="checkpoint directory containing rl_agent_config.json")
+ evidence_cmd.add_argument("--report", default=None,
+ help="an existing laya-evals report JSON")
+ evidence_cmd.add_argument("--json", dest="json_out", default=None,
+ help="write the full inspection JSON to this path")
return parser
@@ -436,6 +444,19 @@ def _cmd_run(args) -> int:
return 0
+def _cmd_evidence(args) -> int:
+ try:
+ result = evidence.inspect_checkpoint(args.checkpoint, args.report)
+ except (FileNotFoundError, ValueError) as exc:
+ print("laya-evals: %s" % exc, file=sys.stderr)
+ return 2
+ print(evidence.format_summary(result))
+ if args.json_out:
+ with open(args.json_out, "w", encoding="utf-8") as handle:
+ json.dump(result, handle, ensure_ascii=False, indent=2, sort_keys=True)
+ return 0
+
+
def _cmd_compare(args) -> int:
# `_identity_of` rather than a bare `config` slice, because a report may carry its identity
# at the top level -- `research/evals/act_head_eval.py` puts `schema` there -- and
@@ -474,6 +495,8 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
return _cmd_validate(args)
if args.command == "run":
return _cmd_run(args)
+ if args.command == "evidence":
+ return _cmd_evidence(args)
return _cmd_compare(args)
except EvalError as exc:
# A malformed dataset, an unreadable report, or a mistyped pin is a usage error, not a
diff --git a/laya/evidence.py b/laya/evidence.py
new file mode 100644
index 00000000..5034b1f9
--- /dev/null
+++ b/laya/evidence.py
@@ -0,0 +1,167 @@
+"""Read-only evidence inspection over existing Laya artifacts.
+
+This module reports what evidence exists for a checkpoint -- and what is missing,
+insufficient, or incomparable -- without loading model weights, without needing
+torch, and without writing a new manifest. It only reads artifacts the upstream
+train/eval path already persists:
+
+- a checkpoint directory's ``rl_agent_config.json`` (``training.laya_train`` and
+ ``training.laya_train_calibration`` as written by merged #931/#933);
+- an existing ``laya-evals`` report JSON (identity semantics from `laya.evals`).
+
+Every state it assigns is either derived directly from an upstream artifact or
+marked UNKNOWN. It never manufactures a healthy-looking state from absent input.
+"""
+from __future__ import annotations
+
+import json
+import os
+from typing import Any, Dict, List, Optional, Tuple
+
+PRESENT = "PRESENT"
+MISSING = "MISSING"
+INSUFFICIENT = "INSUFFICIENT"
+UNKNOWN = "UNKNOWN"
+INCOMPARABLE = "INCOMPARABLE"
+
+# Documented upstream floors/thresholds. `laya.calibrate.MIN_TYPE_N` and
+# `laya.train.CALIB_WARN_N` live in modules that import torch at module scope, so a
+# read-only command cannot import them without pulling torch in. We prefer the live
+# constants when torch is importable and fall back to the documented values with an
+# explicit provenance note when it is not; we never silently duplicate.
+_FALLBACK_MIN_TYPE_N = 10
+_FALLBACK_CALIB_WARN_N = 50
+
+
+def _thresholds() -> Tuple[int, int, str]:
+ try:
+ from .calibrate import MIN_TYPE_N # noqa: F401 (imports torch on success)
+ from .train import CALIB_WARN_N
+ return int(MIN_TYPE_N), int(CALIB_WARN_N), "upstream modules"
+ except Exception:
+ return _FALLBACK_MIN_TYPE_N, _FALLBACK_CALIB_WARN_N, "documented fallback (torch unavailable)"
+
+
+def _entry(state: str, reason: str, detail: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+ out = {"state": state, "reason": reason}
+ if detail:
+ out["detail"] = detail
+ return out
+
+
+def inspect_calibration(config: Dict[str, Any]) -> Dict[str, Any]:
+ """Evidence state of the persisted calibration fit in a checkpoint config."""
+ training = config.get("training")
+ if not isinstance(training, dict):
+ return _entry(UNKNOWN, "legacy checkpoint: no training metadata persisted; cannot tell whether calibration ran")
+ calibration = training.get("laya_train_calibration")
+ if not isinstance(calibration, dict):
+ return _entry(UNKNOWN, "training metadata present but no calibration block; checkpoint predates #933")
+ min_type_n, calib_warn_n, threshold_source = _thresholds()
+ types: Dict[str, Any] = {}
+ for name, entry in calibration.items():
+ if not isinstance(entry, dict) or not isinstance(entry.get("items"), int):
+ types[name] = _entry(UNKNOWN, "calibration entry is malformed")
+ continue
+ items = entry["items"]
+ issues = entry.get("issues") if isinstance(entry.get("issues"), list) else []
+ if items == 0:
+ types[name] = _entry(MISSING, "question type present but zero calibration items", {"items": 0})
+ elif items < min_type_n:
+ types[name] = _entry(INSUFFICIENT, "not fitted: %d items, below MIN_TYPE_N=%d, temperature stays 1.0"
+ % (items, min_type_n), {"items": items, "issues": issues})
+ elif issues:
+ insufficient = [i for i in issues if "fitted on only" in i or "clamp" in i or "unchanged at 1.0" in i]
+ if insufficient:
+ types[name] = _entry(INSUFFICIENT, "fitted with evidence limitations: " + "; ".join(issues),
+ {"items": items, "issues": issues})
+ else:
+ types[name] = _entry(PRESENT, "fitted; issues recorded: " + "; ".join(issues),
+ {"items": items, "issues": issues, "temperature": entry.get("temperature")})
+ elif items < calib_warn_n:
+ types[name] = _entry(INSUFFICIENT, "fitted on only %d items, below CALIB_WARN_N=%d"
+ % (items, calib_warn_n), {"items": items, "issues": issues})
+ else:
+ types[name] = _entry(PRESENT, "fitted on %d items with no recorded issues" % items,
+ {"items": items, "temperature": entry.get("temperature")})
+ present = sum(1 for t in types.values() if t["state"] == PRESENT)
+ overall = PRESENT if types and present == len(types) else (
+ MISSING if types and all(t["state"] == MISSING for t in types.values()) else INSUFFICIENT)
+ return {
+ "state": overall,
+ "threshold_source": threshold_source,
+ "types": types,
+ }
+
+
+def inspect_report(report: Dict[str, Any]) -> Dict[str, Any]:
+ """Evidence state of an eval report's identity fields (reuses EvalReport semantics)."""
+ config = report.get("config")
+ if not isinstance(config, dict):
+ return _entry(UNKNOWN, "report has no config block; identity unknown")
+ fields = {}
+ for key in ("schema", "dataset_sha256", "questions_sha256", "laya_version"):
+ value = config.get(key)
+ if value is None and report.get(key) is not None:
+ value = report.get(key)
+ fields[key] = _entry(PRESENT, "recorded", {"value": value}) if value is not None else _entry(UNKNOWN, "not recorded in this report")
+ identity_keys = [fields["schema"], fields["dataset_sha256"], fields["questions_sha256"]]
+ overall = PRESENT if all(f["state"] == PRESENT for f in identity_keys) else UNKNOWN
+ return {"state": overall, "fields": fields}
+
+
+def relate(config: Dict[str, Any], report: Dict[str, Any]) -> Dict[str, Any]:
+ """Deterministic checkpoint <-> report relationship. Never infers from names/paths."""
+ report_config = report.get("config") if isinstance(report.get("config"), dict) else {}
+ training = config.get("training") if isinstance(config.get("training"), dict) else {}
+ laya_train = training.get("laya_train") if isinstance(training.get("laya_train"), dict) else {}
+ conflicts = []
+ matches = []
+ # Only compare keys that exist on both sides with a real value.
+ for key in ("laya_version",):
+ a = laya_train.get(key) or config.get(key)
+ b = report_config.get(key) or (report.get("config") or {}).get("checkpoint", {}).get("laya")
+ if a is not None and b is not None:
+ (matches if str(a) == str(b) else conflicts).append(key)
+ if conflicts:
+ return _entry(INCOMPARABLE, "checkpoint and report disagree on: " + ", ".join(conflicts))
+ if matches:
+ return _entry(PRESENT, "checkpoint and report agree on: " + ", ".join(matches))
+ return _entry(UNKNOWN, "no shared identity field between this checkpoint config and this report")
+
+
+def inspect_checkpoint(checkpoint_dir: str, report_path: Optional[str] = None) -> Dict[str, Any]:
+ """Full read-only evidence inspection for one checkpoint (optionally one report)."""
+ config_path = os.path.join(checkpoint_dir, "rl_agent_config.json")
+ if not os.path.isdir(checkpoint_dir) or not os.path.isfile(config_path):
+ raise FileNotFoundError("no rl_agent_config.json under %r" % checkpoint_dir)
+ with open(config_path, "r", encoding="utf-8") as handle:
+ config = json.load(handle)
+ if not isinstance(config, dict):
+ raise ValueError("%s is not a JSON object" % config_path)
+ out: Dict[str, Any] = {"checkpoint": checkpoint_dir, "calibration": inspect_calibration(config)}
+ if report_path is not None:
+ with open(report_path, "r", encoding="utf-8") as handle:
+ report = json.load(handle)
+ if not isinstance(report, dict):
+ raise ValueError("%s is not a JSON object" % report_path)
+ out["report"] = inspect_report(report)
+ out["relationship"] = relate(config, report)
+ return out
+
+
+def format_summary(result: Dict[str, Any]) -> str:
+ lines: List[str] = ["evidence summary for %s" % result["checkpoint"]]
+ cal = result["calibration"]
+ lines.append("calibration: %s (thresholds: %s)" % (cal["state"], cal.get("threshold_source", "n/a")))
+ for name, t in (cal.get("types") or {}).items():
+ lines.append(" %-24s %s -- %s" % (name, t["state"], t["reason"]))
+ if cal.get("state") == UNKNOWN and "reason" in cal:
+ lines.append(" " + cal["reason"])
+ if "report" in result:
+ rep = result["report"]
+ lines.append("eval report identity: %s" % rep["state"])
+ for key, f in rep.get("fields", {}).items():
+ lines.append(" %-18s %s" % (key, f["state"]))
+ lines.append("checkpoint<->report: %s -- %s" % (result["relationship"]["state"], result["relationship"]["reason"]))
+ return "\n".join(lines)
diff --git a/tests/test_evals_api.py b/tests/test_evals_api.py
index 2a1c8bc7..5f5b1c27 100644
--- a/tests/test_evals_api.py
+++ b/tests/test_evals_api.py
@@ -156,7 +156,7 @@ def _subparsers(parser):
_subs = _subparsers(evals_cli._build_parser())
-check("laya-evals subcommands", sorted(_subs), ["compare", "run", "validate"])
+check("laya-evals subcommands", sorted(_subs), ["compare", "evidence", "run", "validate"])
_unhelped, _unstated, _options = [], [], {}
for _sub, _parser in sorted(_subs.items()):
diff --git a/tests/test_evidence.py b/tests/test_evidence.py
new file mode 100644
index 00000000..83ddcedb
--- /dev/null
+++ b/tests/test_evidence.py
@@ -0,0 +1,137 @@
+"""Contract guard for laya.evidence: readiness evidence states over existing artifacts.
+
+Run: python tests/test_evidence.py
+"""
+import json
+import os
+import sys
+import tempfile
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from laya import evidence # noqa: E402
+from laya import evals_cli # noqa: E402
+
+PASS, FAIL = [], []
+
+
+def check(name, got, expected):
+ ok = got == expected
+ (PASS if ok else FAIL).append(name)
+ print((" ok " if ok else " FAIL ") + name, "" if ok else "(got %r, expected %r)" % (got, expected))
+
+
+def check_true(name, condition):
+ check(name, bool(condition), True)
+
+
+def calib(config):
+ return evidence.inspect_calibration(config)
+
+
+# ------------------------------------------------------ calibration states
+legacy = calib({"fine_tuned": True})
+check("legacy checkpoint (no training key) -> UNKNOWN", legacy["state"], evidence.UNKNOWN)
+
+legacy2 = calib({"training": {"laya_train": {"base": "x"}}})
+check("training but no calibration block -> UNKNOWN", legacy2["state"], evidence.UNKNOWN)
+
+zero = calib({"training": {"laya_train_calibration": {"choice": {"items": 0, "temperature": 1.0, "issues": []}}}})
+check("zero items -> MISSING", zero["types"]["choice"]["state"], evidence.MISSING)
+
+low = calib({"training": {"laya_train_calibration": {"choice": {"items": 5, "temperature": 1.0,
+ "issues": ["not fitted: 5 calibration items, fewer than 10, so the temperature stays 1.0"]}}}})
+check("below MIN_TYPE_N -> INSUFFICIENT", low["types"]["choice"]["state"], evidence.INSUFFICIENT)
+
+warned = calib({"training": {"laya_train_calibration": {"choice": {"items": 30, "temperature": 1.2,
+ "issues": ["fitted on only 30 calibration items"]}}}})
+check("fitted below CALIB_WARN_N -> INSUFFICIENT", warned["types"]["choice"]["state"], evidence.INSUFFICIENT)
+
+clamped = calib({"training": {"laya_train_calibration": {"choice": {"items": 200, "temperature": 0.5,
+ "issues": ["the fitted temperature 0.5 is on the [0.5, 3] clamp"]}}}})
+check("clamped fit stays a visible limitation -> INSUFFICIENT", clamped["types"]["choice"]["state"], evidence.INSUFFICIENT)
+
+clean = calib({"training": {"laya_train_calibration": {"choice": {"items": 400, "temperature": 1.1, "issues": []}}}})
+check("clean supported fit -> PRESENT", clean["types"]["choice"]["state"], evidence.PRESENT)
+check("all-clean overall -> PRESENT", clean["state"], evidence.PRESENT)
+
+mixed = calib({"training": {"laya_train_calibration": {
+ "choice": {"items": 400, "temperature": 1.1, "issues": []},
+ "score": {"items": 0, "temperature": 1.0, "issues": []}}}})
+check("mixed types overall -> INSUFFICIENT", mixed["state"], evidence.INSUFFICIENT)
+
+malformed = calib({"training": {"laya_train_calibration": {"choice": {"temperature": 1.0}}}})
+check("malformed entry -> UNKNOWN", malformed["types"]["choice"]["state"], evidence.UNKNOWN)
+
+# ------------------------------------------------------ report identity
+full_report = {"config": {"schema": "laya-evals-report/1", "dataset_sha256": "a",
+ "questions_sha256": "b", "laya_version": "0.3.28"}, "overall": {}}
+rep = evidence.inspect_report(full_report)
+check("full identity -> PRESENT", rep["state"], evidence.PRESENT)
+
+old_report = {"config": {"schema": "laya-evals-report/1"}, "overall": {}}
+rep2 = evidence.inspect_report(old_report)
+check("missing identity fields -> UNKNOWN", rep2["state"], evidence.UNKNOWN)
+
+toplevel = {"schema": "laya-evals-report/1", "dataset_sha256": "a", "questions_sha256": "b",
+ "config": {}}
+check("identity at top level still read", evidence.inspect_report(toplevel)["state"], evidence.PRESENT)
+
+# ------------------------------------------------------ relationship
+config = {"training": {"laya_train": {"laya_version": "0.3.28"}}}
+match = evidence.relate(config, {"config": {"laya_version": "0.3.28"}})
+check("agreeing laya_version -> PRESENT", match["state"], evidence.PRESENT)
+conflict = evidence.relate(config, {"config": {"laya_version": "0.3.20"}})
+check("disagreeing laya_version -> INCOMPARABLE", conflict["state"], evidence.INCOMPARABLE)
+unknown = evidence.relate(config, {"config": {}})
+check("no shared identity field -> UNKNOWN", unknown["state"], evidence.UNKNOWN)
+
+# ------------------------------------------------------ CLI on a realistic artifact pair
+tmp = tempfile.mkdtemp()
+ckpt = os.path.join(tmp, "ckpt")
+os.makedirs(ckpt)
+with open(os.path.join(ckpt, "rl_agent_config.json"), "w", encoding="utf-8") as f:
+ json.dump({"training": {"laya_train": {"laya_version": "0.3.28"}, "laya_train_calibration": {
+ "choice": {"items": 400, "temperature": 1.1, "issues": []},
+ "score": {"items": 30, "temperature": 1.0, "issues": ["fitted on only 30 calibration items"]}}}}, f)
+report_path = os.path.join(tmp, "report.json")
+with open(report_path, "w", encoding="utf-8") as f:
+ json.dump({"config": {"schema": "laya-evals-report/1", "dataset_sha256": "a",
+ "questions_sha256": "b", "laya_version": "0.3.28"}, "overall": {}}, f)
+
+rc = evals_cli.main(["evidence", "--checkpoint", ckpt, "--report", report_path])
+check("evidence CLI returns 0 on realistic pair", rc, 0)
+
+rc_bad = evals_cli.main(["evidence", "--checkpoint", os.path.join(tmp, "nope")])
+check("missing checkpoint dir -> usage error 2", rc_bad, 2)
+
+# ------------------------------------------------------ no torch required
+import subprocess
+_BLOCK = (
+ "import sys, os, json, tempfile\n"
+ "sys.path.insert(0, os.getcwd())\n"
+ "class B:\n"
+ " def find_spec(self, name, path, target=None):\n"
+ " if name == 'torch' or name.startswith('torch.'):\n"
+ " raise ImportError('torch blocked')\n"
+ "sys.meta_path.insert(0, B())\n"
+ "tmp = tempfile.mkdtemp(); ckpt = os.path.join(tmp, 'c'); os.makedirs(ckpt)\n"
+ "with open(os.path.join(ckpt, 'rl_agent_config.json'), 'w') as f:\n"
+ " json.dump({'training': {'laya_train_calibration': {'choice': {'items': 5, 'temperature': 1.0, 'issues': []}}}}, f)\n"
+ "from laya import evals_cli, evidence\n"
+ "rc = evals_cli.main(['evidence', '--checkpoint', ckpt])\n"
+ "res = evidence.inspect_checkpoint(ckpt)\n"
+ "print('RC', rc, 'THRESH', res['calibration']['threshold_source'], 'TORCH', 'torch' in sys.modules)\n"
+)
+proc = subprocess.run([sys.executable, "-c", _BLOCK], cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+ capture_output=True, text=True, timeout=120)
+print(proc.stdout.strip(), proc.stderr.strip()[-300:])
+out = proc.stdout
+check_true("evidence CLI works with torch blocked", "RC 0" in out)
+check_true("torch not imported when blocked", "TORCH False" in out)
+check_true("threshold fallback used when torch blocked", "documented fallback" in out)
+
+if FAIL:
+ print("\nFAILURES:", FAIL)
+ sys.exit(1)
+print("\nall %d checks passed" % len(PASS))