Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): conservative checkpoint-report identity, artifact-authoritative calibration issues

Bruce-Yiia735967634cd
diff --git a/docs/evals.md b/docs/evals.md
index 9bcfc862..7940582e 100644
--- a/docs/evals.md
+++ b/docs/evals.md
@@ -405,12 +405,13 @@ laya-evals evidence --checkpoint ./my-checkpoint [--report report.json] [--json
A checkpoint directory must contain `rl_agent_config.json`. Calibration evidence is derived from
the persisted `training.laya_train_calibration` block: a legacy checkpoint with no training
-metadata is `UNKNOWN`; a question type with zero items is `MISSING`; fewer than `MIN_TYPE_N` items
-or a fit with recorded limitations (below `CALIB_WARN_N`, clamped, unchanged at 1.0) is
-`INSUFFICIENT`; a clean supported fit is `PRESENT`. When torch is unavailable, the thresholds
-fall back to the documented `MIN_TYPE_N`/`CALIB_WARN_N` values and the summary says so.
+metadata is `UNKNOWN`; a question type with zero items is `MISSING`; a type whose entry records
+any upstream `issues` text (not fitted, below `MIN_TYPE_N`/`CALIB_WARN_N`, clamped, unchanged
+fit) is `INSUFFICIENT`; a clean fit is `PRESENT`. The helper trusts the persisted `issues`
+written by #933 instead of keeping a second numeric threshold source.
The eval report's identity fields (`schema`, `dataset_sha256`, `questions_sha256`, `laya_version`)
are reported with the same semantics as `run`. The checkpoint↔report relationship is claimed only
-when the shared identity fields provably agree; a real conflict is `INCOMPARABLE`, and shared
-nothing is `UNKNOWN` — never an inference from paths or names.
+as a deterministic conflict: if both artifacts expose the same `dataset_sha256`/`questions_sha256`
+with different values, it is `INCOMPARABLE`. Everything else is `UNKNOWN` — matching `laya_version`
+alone never proves a match, and differing `laya_version` alone never proves a conflict.
diff --git a/laya/evidence.py b/laya/evidence.py
index 5034b1f9..9e97e46d 100644
--- a/laya/evidence.py
+++ b/laya/evidence.py
@@ -16,7 +16,7 @@ from __future__ import annotations
import json
import os
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional
PRESENT = "PRESENT"
MISSING = "MISSING"
@@ -29,19 +29,6 @@ INCOMPARABLE = "INCOMPARABLE"
# read-only command cannot import them without pulling torch in. We prefer the live
# constants when torch is importable and fall back to the documented values with an
# explicit provenance note when it is not; we never silently duplicate.
-_FALLBACK_MIN_TYPE_N = 10
-_FALLBACK_CALIB_WARN_N = 50
-
-
-def _thresholds() -> Tuple[int, int, str]:
- try:
- from .calibrate import MIN_TYPE_N # noqa: F401 (imports torch on success)
- from .train import CALIB_WARN_N
- return int(MIN_TYPE_N), int(CALIB_WARN_N), "upstream modules"
- except Exception:
- return _FALLBACK_MIN_TYPE_N, _FALLBACK_CALIB_WARN_N, "documented fallback (torch unavailable)"
-
-
def _entry(state: str, reason: str, detail: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
out = {"state": state, "reason": reason}
if detail:
@@ -57,39 +44,32 @@ def inspect_calibration(config: Dict[str, Any]) -> Dict[str, Any]:
calibration = training.get("laya_train_calibration")
if not isinstance(calibration, dict):
return _entry(UNKNOWN, "training metadata present but no calibration block; checkpoint predates #933")
- min_type_n, calib_warn_n, threshold_source = _thresholds()
types: Dict[str, Any] = {}
for name, entry in calibration.items():
if not isinstance(entry, dict) or not isinstance(entry.get("items"), int):
types[name] = _entry(UNKNOWN, "calibration entry is malformed")
continue
items = entry["items"]
- issues = entry.get("issues") if isinstance(entry.get("issues"), list) else []
if items == 0:
types[name] = _entry(MISSING, "question type present but zero calibration items", {"items": 0})
- elif items < min_type_n:
- types[name] = _entry(INSUFFICIENT, "not fitted: %d items, below MIN_TYPE_N=%d, temperature stays 1.0"
- % (items, min_type_n), {"items": items, "issues": issues})
+ continue
+ # Persisted issues are the authoritative limitation text (#933 writes
+ # not-fitted/below-floor/clamped/unchanged-fit there); we do not keep a
+ # second numeric source of truth.
+ issues = entry.get("issues")
+ if not isinstance(issues, list):
+ types[name] = _entry(UNKNOWN, "calibration entry has no issues list; cannot tell limitations")
elif issues:
- insufficient = [i for i in issues if "fitted on only" in i or "clamp" in i or "unchanged at 1.0" in i]
- if insufficient:
- types[name] = _entry(INSUFFICIENT, "fitted with evidence limitations: " + "; ".join(issues),
- {"items": items, "issues": issues})
- else:
- types[name] = _entry(PRESENT, "fitted; issues recorded: " + "; ".join(issues),
- {"items": items, "issues": issues, "temperature": entry.get("temperature")})
- elif items < calib_warn_n:
- types[name] = _entry(INSUFFICIENT, "fitted on only %d items, below CALIB_WARN_N=%d"
- % (items, calib_warn_n), {"items": items, "issues": issues})
+ types[name] = _entry(INSUFFICIENT, "fitted with recorded limitations: " + "; ".join(str(i) for i in issues),
+ {"items": items, "issues": issues})
else:
- types[name] = _entry(PRESENT, "fitted on %d items with no recorded issues" % items,
+ types[name] = _entry(PRESENT, "fitted with no recorded issues on %d items" % items,
{"items": items, "temperature": entry.get("temperature")})
present = sum(1 for t in types.values() if t["state"] == PRESENT)
overall = PRESENT if types and present == len(types) else (
MISSING if types and all(t["state"] == MISSING for t in types.values()) else INSUFFICIENT)
return {
"state": overall,
- "threshold_source": threshold_source,
"types": types,
}
@@ -110,24 +90,41 @@ def inspect_report(report: Dict[str, Any]) -> Dict[str, Any]:
return {"state": overall, "fields": fields}
+_IDENTITY_CONFLICT_KEYS = ("dataset_sha256", "questions_sha256")
+
+
+def _field(config_like: Any, key: str) -> Any:
+ if not isinstance(config_like, dict):
+ return None
+ if config_like.get(key) is not None:
+ return config_like.get(key)
+ nested = config_like.get("training")
+ if isinstance(nested, dict):
+ for inner in (nested.get("laya_train"),):
+ if isinstance(inner, dict) and inner.get(key) is not None:
+ return inner.get(key)
+ return None
+
+
def relate(config: Dict[str, Any], report: Dict[str, Any]) -> Dict[str, Any]:
- """Deterministic checkpoint <-> report relationship. Never infers from names/paths."""
+ """Deterministic checkpoint <-> report relationship.
+
+ v1 is deliberately conservative: it emits INCOMPARABLE only when both artifacts
+ expose the same deterministic identity key with conflicting values, and it never
+ infers a PRESENT match from `laya_version` (which names the software, not the
+ checkpoint). Everything else is UNKNOWN -- current artifacts expose no immutable
+ shared checkpoint identity.
+ """
report_config = report.get("config") if isinstance(report.get("config"), dict) else {}
- training = config.get("training") if isinstance(config.get("training"), dict) else {}
- laya_train = training.get("laya_train") if isinstance(training.get("laya_train"), dict) else {}
conflicts = []
- matches = []
- # Only compare keys that exist on both sides with a real value.
- for key in ("laya_version",):
- a = laya_train.get(key) or config.get(key)
- b = report_config.get(key) or (report.get("config") or {}).get("checkpoint", {}).get("laya")
- if a is not None and b is not None:
- (matches if str(a) == str(b) else conflicts).append(key)
+ for key in _IDENTITY_CONFLICT_KEYS:
+ a = _field(config, key)
+ b = _field(report_config, key)
+ if a is not None and b is not None and str(a) != str(b):
+ conflicts.append(key)
if conflicts:
return _entry(INCOMPARABLE, "checkpoint and report disagree on: " + ", ".join(conflicts))
- if matches:
- return _entry(PRESENT, "checkpoint and report agree on: " + ", ".join(matches))
- return _entry(UNKNOWN, "no shared identity field between this checkpoint config and this report")
+ return _entry(UNKNOWN, "no immutable shared checkpoint identity between this checkpoint artifact and the report")
def inspect_checkpoint(checkpoint_dir: str, report_path: Optional[str] = None) -> Dict[str, Any]:
@@ -153,7 +150,7 @@ def inspect_checkpoint(checkpoint_dir: str, report_path: Optional[str] = None) -
def format_summary(result: Dict[str, Any]) -> str:
lines: List[str] = ["evidence summary for %s" % result["checkpoint"]]
cal = result["calibration"]
- lines.append("calibration: %s (thresholds: %s)" % (cal["state"], cal.get("threshold_source", "n/a")))
+ lines.append("calibration: %s" % cal["state"])
for name, t in (cal.get("types") or {}).items():
lines.append(" %-24s %s -- %s" % (name, t["state"], t["reason"]))
if cal.get("state") == UNKNOWN and "reason" in cal:
diff --git a/tests/test_evidence.py b/tests/test_evidence.py
index 83ddcedb..11e1458a 100644
--- a/tests/test_evidence.py
+++ b/tests/test_evidence.py
@@ -80,9 +80,11 @@ check("identity at top level still read", evidence.inspect_report(toplevel)["sta
# ------------------------------------------------------ relationship
config = {"training": {"laya_train": {"laya_version": "0.3.28"}}}
match = evidence.relate(config, {"config": {"laya_version": "0.3.28"}})
-check("agreeing laya_version -> PRESENT", match["state"], evidence.PRESENT)
+check("same laya_version alone must NOT prove PRESENT", match["state"], evidence.UNKNOWN)
conflict = evidence.relate(config, {"config": {"laya_version": "0.3.20"}})
-check("disagreeing laya_version -> INCOMPARABLE", conflict["state"], evidence.INCOMPARABLE)
+check("different laya_version must NOT become INCOMPARABLE", conflict["state"], evidence.UNKNOWN)
+conflict_ds = evidence.relate({"dataset_sha256": "a"}, {"config": {"dataset_sha256": "b"}})
+check("deterministic identity conflict -> INCOMPARABLE", conflict_ds["state"], evidence.INCOMPARABLE)
unknown = evidence.relate(config, {"config": {}})
check("no shared identity field -> UNKNOWN", unknown["state"], evidence.UNKNOWN)
@@ -121,7 +123,7 @@ _BLOCK = (
"from laya import evals_cli, evidence\n"
"rc = evals_cli.main(['evidence', '--checkpoint', ckpt])\n"
"res = evidence.inspect_checkpoint(ckpt)\n"
- "print('RC', rc, 'THRESH', res['calibration']['threshold_source'], 'TORCH', 'torch' in sys.modules)\n"
+ "print('RC', rc, 'TORCH', 'torch' in sys.modules)\n"
)
proc = subprocess.run([sys.executable, "-c", _BLOCK], cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
capture_output=True, text=True, timeout=120)
@@ -129,7 +131,6 @@ print(proc.stdout.strip(), proc.stderr.strip()[-300:])
out = proc.stdout
check_true("evidence CLI works with torch blocked", "RC 0" in out)
check_true("torch not imported when blocked", "TORCH False" in out)
-check_true("threshold fallback used when torch blocked", "documented fallback" in out)
if FAIL:
print("\nFAILURES:", FAIL)