Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(evals): read-only evidence inspection over checkpoint and eval artifacts
Bruce-Yii473a35f335be
diff --git a/docs/evals.md b/docs/evals.mdindex 8a65d0ec..9bcfc862 100644--- a/docs/evals.md+++ b/docs/evals.md@@ -392,3 +392,25 @@ opt in. With `--onnx`, only a bare `--revision <SHA>` applies, to the config andDrop a JSONL in `research/evals/` and a reviewed baseline beside it, then point a workflow (or`research/evals/check_regression.py`) at both. The format is the same as the fixture; nothing inthe harness knows about MASSIVE.++## Evidence inspection++`evidence` reads — never writes — the artifacts a fine-tune/eval run already persists and says+what evidence exists, what is missing, and what is insufficient. It loads no model weights and+does not require torch.++```bash+laya-evals evidence --checkpoint ./my-checkpoint [--report report.json] [--json out.json]+```++A checkpoint directory must contain `rl_agent_config.json`. Calibration evidence is derived from+the persisted `training.laya_train_calibration` block: a legacy checkpoint with no training+metadata is `UNKNOWN`; a question type with zero items is `MISSING`; fewer than `MIN_TYPE_N` items+or a fit with recorded limitations (below `CALIB_WARN_N`, clamped, unchanged at 1.0) is+`INSUFFICIENT`; a clean supported fit is `PRESENT`. When torch is unavailable, the thresholds+fall back to the documented `MIN_TYPE_N`/`CALIB_WARN_N` values and the summary says so.++The eval report's identity fields (`schema`, `dataset_sha256`, `questions_sha256`, `laya_version`)+are reported with the same semantics as `run`. The checkpoint↔report relationship is claimed only+when the shared identity fields provably agree; a real conflict is `INCOMPARABLE`, and shared+nothing is `UNKNOWN` — never an inference from paths or names.diff --git a/laya/evals_cli.py b/laya/evals_cli.pyindex 66897dc1..3bb9b4ea 100644--- a/laya/evals_cli.py+++ b/laya/evals_cli.py@@ -17,7 +17,7 @@ import mathimport sysfrom typing import Any, Dict, List, Optional, Sequence, Tuple-from . import _eval_policy, evals+from . import _eval_policy, evals, evidencefrom .evals import EvalError@@ -230,6 +230,14 @@ def _build_parser() -> argparse.ArgumentParser:help="allowed absolute drift; repeatable")compare.add_argument("--gate-policy", metavar="FILE",help="apply opt-in per-slice quality rules from a JSON policy")+ evidence_cmd = sub.add_parser("evidence",+ help="read-only evidence inspection over a checkpoint config and an eval report")+ evidence_cmd.add_argument("--checkpoint", required=True,+ help="checkpoint directory containing rl_agent_config.json")+ evidence_cmd.add_argument("--report", default=None,+ help="an existing laya-evals report JSON")+ evidence_cmd.add_argument("--json", dest="json_out", default=None,+ help="write the full inspection JSON to this path")return parser@@ -436,6 +444,19 @@ def _cmd_run(args) -> int:return 0+def _cmd_evidence(args) -> int:+ try:+ result = evidence.inspect_checkpoint(args.checkpoint, args.report)+ except (FileNotFoundError, ValueError) as exc:+ print("laya-evals: %s" % exc, file=sys.stderr)+ return 2+ print(evidence.format_summary(result))+ if args.json_out:+ with open(args.json_out, "w", encoding="utf-8") as handle:+ json.dump(result, handle, ensure_ascii=False, indent=2, sort_keys=True)+ return 0++def _cmd_compare(args) -> int:# `_identity_of` rather than a bare `config` slice, because a report may carry its identity# at the top level -- `research/evals/act_head_eval.py` puts `schema` there -- and@@ -474,6 +495,8 @@ def main(argv: Optional[Sequence[str]] = None) -> int:return _cmd_validate(args)if args.command == "run":return _cmd_run(args)+ if args.command == "evidence":+ return _cmd_evidence(args)return _cmd_compare(args)except EvalError as exc:# A malformed dataset, an unreadable report, or a mistyped pin is a usage error, not adiff --git a/laya/evidence.py b/laya/evidence.pynew file mode 100644index 00000000..5034b1f9--- /dev/null+++ b/laya/evidence.py@@ -0,0 +1,167 @@+"""Read-only evidence inspection over existing Laya artifacts.++This module reports what evidence exists for a checkpoint -- and what is missing,+insufficient, or incomparable -- without loading model weights, without needing+torch, and without writing a new manifest. It only reads artifacts the upstream+train/eval path already persists:++- a checkpoint directory's ``rl_agent_config.json`` (``training.laya_train`` and+ ``training.laya_train_calibration`` as written by merged #931/#933);+- an existing ``laya-evals`` report JSON (identity semantics from `laya.evals`).++Every state it assigns is either derived directly from an upstream artifact or+marked UNKNOWN. It never manufactures a healthy-looking state from absent input.+"""+from __future__ import annotations++import json+import os+from typing import Any, Dict, List, Optional, Tuple++PRESENT = "PRESENT"+MISSING = "MISSING"+INSUFFICIENT = "INSUFFICIENT"+UNKNOWN = "UNKNOWN"+INCOMPARABLE = "INCOMPARABLE"++# Documented upstream floors/thresholds. `laya.calibrate.MIN_TYPE_N` and+# `laya.train.CALIB_WARN_N` live in modules that import torch at module scope, so a+# read-only command cannot import them without pulling torch in. We prefer the live+# constants when torch is importable and fall back to the documented values with an+# explicit provenance note when it is not; we never silently duplicate.+_FALLBACK_MIN_TYPE_N = 10+_FALLBACK_CALIB_WARN_N = 50+++def _thresholds() -> Tuple[int, int, str]:+ try:+ from .calibrate import MIN_TYPE_N # noqa: F401 (imports torch on success)+ from .train import CALIB_WARN_N+ return int(MIN_TYPE_N), int(CALIB_WARN_N), "upstream modules"+ except Exception:+ return _FALLBACK_MIN_TYPE_N, _FALLBACK_CALIB_WARN_N, "documented fallback (torch unavailable)"+++def _entry(state: str, reason: str, detail: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:+ out = {"state": state, "reason": reason}+ if detail:+ out["detail"] = detail+ return out+++def inspect_calibration(config: Dict[str, Any]) -> Dict[str, Any]:+ """Evidence state of the persisted calibration fit in a checkpoint config."""+ training = config.get("training")+ if not isinstance(training, dict):+ return _entry(UNKNOWN, "legacy checkpoint: no training metadata persisted; cannot tell whether calibration ran")+ calibration = training.get("laya_train_calibration")+ if not isinstance(calibration, dict):+ return _entry(UNKNOWN, "training metadata present but no calibration block; checkpoint predates #933")+ min_type_n, calib_warn_n, threshold_source = _thresholds()+ types: Dict[str, Any] = {}+ for name, entry in calibration.items():+ if not isinstance(entry, dict) or not isinstance(entry.get("items"), int):+ types[name] = _entry(UNKNOWN, "calibration entry is malformed")+ continue+ items = entry["items"]+ issues = entry.get("issues") if isinstance(entry.get("issues"), list) else []+ if items == 0:+ types[name] = _entry(MISSING, "question type present but zero calibration items", {"items": 0})+ elif items < min_type_n:+ types[name] = _entry(INSUFFICIENT, "not fitted: %d items, below MIN_TYPE_N=%d, temperature stays 1.0"+ % (items, min_type_n), {"items": items, "issues": issues})+ elif issues:+ insufficient = [i for i in issues if "fitted on only" in i or "clamp" in i or "unchanged at 1.0" in i]+ if insufficient:+ types[name] = _entry(INSUFFICIENT, "fitted with evidence limitations: " + "; ".join(issues),+ {"items": items, "issues": issues})+ else:+ types[name] = _entry(PRESENT, "fitted; issues recorded: " + "; ".join(issues),+ {"items": items, "issues": issues, "temperature": entry.get("temperature")})+ elif items < calib_warn_n:+ types[name] = _entry(INSUFFICIENT, "fitted on only %d items, below CALIB_WARN_N=%d"+ % (items, calib_warn_n), {"items": items, "issues": issues})+ else:+ types[name] = _entry(PRESENT, "fitted on %d items with no recorded issues" % items,+ {"items": items, "temperature": entry.get("temperature")})+ present = sum(1 for t in types.values() if t["state"] == PRESENT)+ overall = PRESENT if types and present == len(types) else (+ MISSING if types and all(t["state"] == MISSING for t in types.values()) else INSUFFICIENT)+ return {+ "state": overall,+ "threshold_source": threshold_source,+ "types": types,+ }+++def inspect_report(report: Dict[str, Any]) -> Dict[str, Any]:+ """Evidence state of an eval report's identity fields (reuses EvalReport semantics)."""+ config = report.get("config")+ if not isinstance(config, dict):+ return _entry(UNKNOWN, "report has no config block; identity unknown")+ fields = {}+ for key in ("schema", "dataset_sha256", "questions_sha256", "laya_version"):+ value = config.get(key)+ if value is None and report.get(key) is not None:+ value = report.get(key)+ fields[key] = _entry(PRESENT, "recorded", {"value": value}) if value is not None else _entry(UNKNOWN, "not recorded in this report")+ identity_keys = [fields["schema"], fields["dataset_sha256"], fields["questions_sha256"]]+ overall = PRESENT if all(f["state"] == PRESENT for f in identity_keys) else UNKNOWN+ return {"state": overall, "fields": fields}+++def relate(config: Dict[str, Any], report: Dict[str, Any]) -> Dict[str, Any]:+ """Deterministic checkpoint <-> report relationship. Never infers from names/paths."""+ report_config = report.get("config") if isinstance(report.get("config"), dict) else {}+ training = config.get("training") if isinstance(config.get("training"), dict) else {}+ laya_train = training.get("laya_train") if isinstance(training.get("laya_train"), dict) else {}+ conflicts = []+ matches = []+ # Only compare keys that exist on both sides with a real value.+ for key in ("laya_version",):+ a = laya_train.get(key) or config.get(key)+ b = report_config.get(key) or (report.get("config") or {}).get("checkpoint", {}).get("laya")+ if a is not None and b is not None:+ (matches if str(a) == str(b) else conflicts).append(key)+ if conflicts:+ return _entry(INCOMPARABLE, "checkpoint and report disagree on: " + ", ".join(conflicts))+ if matches:+ return _entry(PRESENT, "checkpoint and report agree on: " + ", ".join(matches))+ return _entry(UNKNOWN, "no shared identity field between this checkpoint config and this report")+++def inspect_checkpoint(checkpoint_dir: str, report_path: Optional[str] = None) -> Dict[str, Any]:+ """Full read-only evidence inspection for one checkpoint (optionally one report)."""+ config_path = os.path.join(checkpoint_dir, "rl_agent_config.json")+ if not os.path.isdir(checkpoint_dir) or not os.path.isfile(config_path):+ raise FileNotFoundError("no rl_agent_config.json under %r" % checkpoint_dir)+ with open(config_path, "r", encoding="utf-8") as handle:+ config = json.load(handle)+ if not isinstance(config, dict):+ raise ValueError("%s is not a JSON object" % config_path)+ out: Dict[str, Any] = {"checkpoint": checkpoint_dir, "calibration": inspect_calibration(config)}+ if report_path is not None:+ with open(report_path, "r", encoding="utf-8") as handle:+ report = json.load(handle)+ if not isinstance(report, dict):+ raise ValueError("%s is not a JSON object" % report_path)+ out["report"] = inspect_report(report)+ out["relationship"] = relate(config, report)+ return out+++def format_summary(result: Dict[str, Any]) -> str:+ lines: List[str] = ["evidence summary for %s" % result["checkpoint"]]+ cal = result["calibration"]+ lines.append("calibration: %s (thresholds: %s)" % (cal["state"], cal.get("threshold_source", "n/a")))+ for name, t in (cal.get("types") or {}).items():+ lines.append(" %-24s %s -- %s" % (name, t["state"], t["reason"]))+ if cal.get("state") == UNKNOWN and "reason" in cal:+ lines.append(" " + cal["reason"])+ if "report" in result:+ rep = result["report"]+ lines.append("eval report identity: %s" % rep["state"])+ for key, f in rep.get("fields", {}).items():+ lines.append(" %-18s %s" % (key, f["state"]))+ lines.append("checkpoint<->report: %s -- %s" % (result["relationship"]["state"], result["relationship"]["reason"]))+ return "\n".join(lines)diff --git a/tests/test_evals_api.py b/tests/test_evals_api.pyindex 2a1c8bc7..5f5b1c27 100644--- a/tests/test_evals_api.py+++ b/tests/test_evals_api.py@@ -156,7 +156,7 @@ def _subparsers(parser):_subs = _subparsers(evals_cli._build_parser())-check("laya-evals subcommands", sorted(_subs), ["compare", "run", "validate"])+check("laya-evals subcommands", sorted(_subs), ["compare", "evidence", "run", "validate"])_unhelped, _unstated, _options = [], [], {}for _sub, _parser in sorted(_subs.items()):diff --git a/tests/test_evidence.py b/tests/test_evidence.pynew file mode 100644index 00000000..83ddcedb--- /dev/null+++ b/tests/test_evidence.py@@ -0,0 +1,137 @@+"""Contract guard for laya.evidence: readiness evidence states over existing artifacts.++Run: python tests/test_evidence.py+"""+import json+import os+import sys+import tempfile++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from laya import evidence # noqa: E402+from laya import evals_cli # noqa: E402++PASS, FAIL = [], []+++def check(name, got, expected):+ ok = got == expected+ (PASS if ok else FAIL).append(name)+ print((" ok " if ok else " FAIL ") + name, "" if ok else "(got %r, expected %r)" % (got, expected))+++def check_true(name, condition):+ check(name, bool(condition), True)+++def calib(config):+ return evidence.inspect_calibration(config)+++# ------------------------------------------------------ calibration states+legacy = calib({"fine_tuned": True})+check("legacy checkpoint (no training key) -> UNKNOWN", legacy["state"], evidence.UNKNOWN)++legacy2 = calib({"training": {"laya_train": {"base": "x"}}})+check("training but no calibration block -> UNKNOWN", legacy2["state"], evidence.UNKNOWN)++zero = calib({"training": {"laya_train_calibration": {"choice": {"items": 0, "temperature": 1.0, "issues": []}}}})+check("zero items -> MISSING", zero["types"]["choice"]["state"], evidence.MISSING)++low = calib({"training": {"laya_train_calibration": {"choice": {"items": 5, "temperature": 1.0,+ "issues": ["not fitted: 5 calibration items, fewer than 10, so the temperature stays 1.0"]}}}})+check("below MIN_TYPE_N -> INSUFFICIENT", low["types"]["choice"]["state"], evidence.INSUFFICIENT)++warned = calib({"training": {"laya_train_calibration": {"choice": {"items": 30, "temperature": 1.2,+ "issues": ["fitted on only 30 calibration items"]}}}})+check("fitted below CALIB_WARN_N -> INSUFFICIENT", warned["types"]["choice"]["state"], evidence.INSUFFICIENT)++clamped = calib({"training": {"laya_train_calibration": {"choice": {"items": 200, "temperature": 0.5,+ "issues": ["the fitted temperature 0.5 is on the [0.5, 3] clamp"]}}}})+check("clamped fit stays a visible limitation -> INSUFFICIENT", clamped["types"]["choice"]["state"], evidence.INSUFFICIENT)++clean = calib({"training": {"laya_train_calibration": {"choice": {"items": 400, "temperature": 1.1, "issues": []}}}})+check("clean supported fit -> PRESENT", clean["types"]["choice"]["state"], evidence.PRESENT)+check("all-clean overall -> PRESENT", clean["state"], evidence.PRESENT)++mixed = calib({"training": {"laya_train_calibration": {+ "choice": {"items": 400, "temperature": 1.1, "issues": []},+ "score": {"items": 0, "temperature": 1.0, "issues": []}}}})+check("mixed types overall -> INSUFFICIENT", mixed["state"], evidence.INSUFFICIENT)++malformed = calib({"training": {"laya_train_calibration": {"choice": {"temperature": 1.0}}}})+check("malformed entry -> UNKNOWN", malformed["types"]["choice"]["state"], evidence.UNKNOWN)++# ------------------------------------------------------ report identity+full_report = {"config": {"schema": "laya-evals-report/1", "dataset_sha256": "a",+ "questions_sha256": "b", "laya_version": "0.3.28"}, "overall": {}}+rep = evidence.inspect_report(full_report)+check("full identity -> PRESENT", rep["state"], evidence.PRESENT)++old_report = {"config": {"schema": "laya-evals-report/1"}, "overall": {}}+rep2 = evidence.inspect_report(old_report)+check("missing identity fields -> UNKNOWN", rep2["state"], evidence.UNKNOWN)++toplevel = {"schema": "laya-evals-report/1", "dataset_sha256": "a", "questions_sha256": "b",+ "config": {}}+check("identity at top level still read", evidence.inspect_report(toplevel)["state"], evidence.PRESENT)++# ------------------------------------------------------ relationship+config = {"training": {"laya_train": {"laya_version": "0.3.28"}}}+match = evidence.relate(config, {"config": {"laya_version": "0.3.28"}})+check("agreeing laya_version -> PRESENT", match["state"], evidence.PRESENT)+conflict = evidence.relate(config, {"config": {"laya_version": "0.3.20"}})+check("disagreeing laya_version -> INCOMPARABLE", conflict["state"], evidence.INCOMPARABLE)+unknown = evidence.relate(config, {"config": {}})+check("no shared identity field -> UNKNOWN", unknown["state"], evidence.UNKNOWN)++# ------------------------------------------------------ CLI on a realistic artifact pair+tmp = tempfile.mkdtemp()+ckpt = os.path.join(tmp, "ckpt")+os.makedirs(ckpt)+with open(os.path.join(ckpt, "rl_agent_config.json"), "w", encoding="utf-8") as f:+ json.dump({"training": {"laya_train": {"laya_version": "0.3.28"}, "laya_train_calibration": {+ "choice": {"items": 400, "temperature": 1.1, "issues": []},+ "score": {"items": 30, "temperature": 1.0, "issues": ["fitted on only 30 calibration items"]}}}}, f)+report_path = os.path.join(tmp, "report.json")+with open(report_path, "w", encoding="utf-8") as f:+ json.dump({"config": {"schema": "laya-evals-report/1", "dataset_sha256": "a",+ "questions_sha256": "b", "laya_version": "0.3.28"}, "overall": {}}, f)++rc = evals_cli.main(["evidence", "--checkpoint", ckpt, "--report", report_path])+check("evidence CLI returns 0 on realistic pair", rc, 0)++rc_bad = evals_cli.main(["evidence", "--checkpoint", os.path.join(tmp, "nope")])+check("missing checkpoint dir -> usage error 2", rc_bad, 2)++# ------------------------------------------------------ no torch required+import subprocess+_BLOCK = (+ "import sys, os, json, tempfile\n"+ "sys.path.insert(0, os.getcwd())\n"+ "class B:\n"+ " def find_spec(self, name, path, target=None):\n"+ " if name == 'torch' or name.startswith('torch.'):\n"+ " raise ImportError('torch blocked')\n"+ "sys.meta_path.insert(0, B())\n"+ "tmp = tempfile.mkdtemp(); ckpt = os.path.join(tmp, 'c'); os.makedirs(ckpt)\n"+ "with open(os.path.join(ckpt, 'rl_agent_config.json'), 'w') as f:\n"+ " json.dump({'training': {'laya_train_calibration': {'choice': {'items': 5, 'temperature': 1.0, 'issues': []}}}}, f)\n"+ "from laya import evals_cli, evidence\n"+ "rc = evals_cli.main(['evidence', '--checkpoint', ckpt])\n"+ "res = evidence.inspect_checkpoint(ckpt)\n"+ "print('RC', rc, 'THRESH', res['calibration']['threshold_source'], 'TORCH', 'torch' in sys.modules)\n"+)+proc = subprocess.run([sys.executable, "-c", _BLOCK], cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))),+ capture_output=True, text=True, timeout=120)+print(proc.stdout.strip(), proc.stderr.strip()[-300:])+out = proc.stdout+check_true("evidence CLI works with torch blocked", "RC 0" in out)+check_true("torch not imported when blocked", "TORCH False" in out)+check_true("threshold fallback used when torch blocked", "documented fallback" in out)++if FAIL:+ print("\nFAILURES:", FAIL)+ sys.exit(1)+print("\nall %d checks passed" % len(PASS))