Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

test(evidence): pin the copied report-identity vocabulary against laya.evals

Aashish4ff895a39a25
diff --git a/laya/evidence.py b/laya/evidence.py
index 375a748f..f88d4b8e 100644
--- a/laya/evidence.py
+++ b/laya/evidence.py
@@ -7,7 +7,8 @@ train/eval path already persists:
- a checkpoint directory's ``rl_agent_config.json`` (``training.laya_train`` and
``training.laya_train_calibration`` as written by merged #931/#933);
-- an existing ``laya-evals`` report JSON (identity semantics from `laya.evals`).
+- an existing ``laya-evals`` report JSON (its identity vocabulary is a copy of `laya.evals`' own,
+ pinned against it by `tests/test_evidence.py`).
Every state it assigns is either derived directly from an upstream artifact or
marked UNKNOWN. It never manufactures a healthy-looking state from absent input.
@@ -73,25 +74,39 @@ def inspect_calibration(config: Dict[str, Any]) -> Dict[str, Any]:
}
+#: The report-identity vocabulary, copied from `laya.evals._IDENTITY_KEYS`. The copy is deliberate:
+#: `laya.evals` imports numpy at module scope, and this module reads a checkpoint directory and a
+#: report JSON with nothing but `json` and `os` on its import path. That is also why
+#: `tests/test_evidence.py` holds the copy against the original in both directions -- a key renamed
+#: upstream would make every report read as identity-incomplete here, and a key added upstream would
+#: read as complete, and neither would fail anything else.
+_IDENTITY_KEYS = ("schema", "dataset_sha256", "questions_sha256")
+
+#: Read as provenance, never required for the identity verdict, because `laya.evals` records
+#: `laya_version` in `config` and deliberately does not compare it
+#: (`laya/evals.py::EvalReport.comparable_to`): a patch release must not invalidate a baseline.
+_PROVENANCE_KEYS = ("laya_version",)
+
+#: The part of that vocabulary two artifacts can actually disagree on. `schema` tags the report, not
+#: the checkpoint, so it is identity for a report and never a checkpoint <-> report conflict.
+_IDENTITY_CONFLICT_KEYS = ("dataset_sha256", "questions_sha256")
+
+
def inspect_report(report: Dict[str, Any]) -> Dict[str, Any]:
- """Evidence state of an eval report's identity fields (reuses EvalReport semantics)."""
+ """Evidence state of an eval report's identity fields, on the vocabulary declared above."""
config = report.get("config")
if not isinstance(config, dict):
return _entry(UNKNOWN, "report has no config block; identity unknown")
fields = {}
- for key in ("schema", "dataset_sha256", "questions_sha256", "laya_version"):
+ for key in _IDENTITY_KEYS + _PROVENANCE_KEYS:
value = config.get(key)
if value is None and report.get(key) is not None:
value = report.get(key)
fields[key] = _entry(PRESENT, "recorded", {"value": value}) if value is not None else _entry(UNKNOWN, "not recorded in this report")
- identity_keys = [fields["schema"], fields["dataset_sha256"], fields["questions_sha256"]]
- overall = PRESENT if all(f["state"] == PRESENT for f in identity_keys) else UNKNOWN
+ overall = PRESENT if all(fields[key]["state"] == PRESENT for key in _IDENTITY_KEYS) else UNKNOWN
return {"state": overall, "fields": fields}
-_IDENTITY_CONFLICT_KEYS = ("dataset_sha256", "questions_sha256")
-
-
def _field(config_like: Any, key: str) -> Any:
if not isinstance(config_like, dict):
return None
diff --git a/tests/test_evidence.py b/tests/test_evidence.py
index da9b6fbf..5c28f67f 100644
--- a/tests/test_evidence.py
+++ b/tests/test_evidence.py
@@ -107,6 +107,33 @@ toplevel = {"schema": "laya-evals-report/1", "dataset_sha256": "a", "questions_s
"config": {}}
check("identity at top level still read", evidence.inspect_report(toplevel)["state"], evidence.PRESENT)
+# ------------------------------------------------------ identity vocabulary pinned to laya.evals
+# `laya/evidence.py` copies the report-identity vocabulary instead of importing `laya.evals` (numpy at
+# module scope), and a copy is only safe while it is held against the original -- the same way
+# `tests/test_conformal_abstention.py` holds `confidence.BUCKET_QTYPES` against `common.temp_bucket`.
+# Unpinned, a key renamed upstream would make every report read as identity-incomplete here, and a key
+# added upstream would read as identity-complete; both silently, in the verdict this module exists to
+# give. These four checks fail in both directions, on either side of the copy.
+from laya import evals # noqa: E402
+
+check("the copied identity vocabulary is evals' own, in order",
+ list(evidence._IDENTITY_KEYS), list(evals._IDENTITY_KEYS))
+check("the provenance key is recorded upstream but not compared there",
+ [k for k in evidence._PROVENANCE_KEYS if k not in evals._IDENTITY_KEYS],
+ list(evidence._PROVENANCE_KEYS))
+check("checkpoint <-> report conflicts are the identity vocabulary minus the report tag",
+ sorted(set(evals._IDENTITY_KEYS) - set(evidence._IDENTITY_CONFLICT_KEYS)), ["schema"])
+check("inspect_report reads exactly its declared vocabulary",
+ sorted(evidence.inspect_report({"config": {}})["fields"]),
+ sorted(evidence._IDENTITY_KEYS + evidence._PROVENANCE_KEYS))
+
+# ... and `schema` stays out of the conflict set on purpose: two artifacts that agree on the
+# deterministic digests but carry different report tags are still the same measurement.
+check("a differing schema is not a checkpoint <-> report conflict",
+ evidence.relate({"dataset_sha256": "a"},
+ {"config": {"dataset_sha256": "a", "schema": "laya-evals-report/2"}})["state"],
+ evidence.UNKNOWN)
+
# ------------------------------------------------------ relationship
config = {"training": {"laya_train": {"laya_version": "0.3.28"}}}
match = evidence.relate(config, {"config": {"laya_version": "0.3.28"}})