Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
test(evidence): pin the copied report-identity vocabulary against laya.evals
Aashish4ff895a39a25
diff --git a/laya/evidence.py b/laya/evidence.pyindex 375a748f..f88d4b8e 100644--- a/laya/evidence.py+++ b/laya/evidence.py@@ -7,7 +7,8 @@ train/eval path already persists:- a checkpoint directory's ``rl_agent_config.json`` (``training.laya_train`` and``training.laya_train_calibration`` as written by merged #931/#933);-- an existing ``laya-evals`` report JSON (identity semantics from `laya.evals`).+- an existing ``laya-evals`` report JSON (its identity vocabulary is a copy of `laya.evals`' own,+ pinned against it by `tests/test_evidence.py`).Every state it assigns is either derived directly from an upstream artifact ormarked UNKNOWN. It never manufactures a healthy-looking state from absent input.@@ -73,25 +74,39 @@ def inspect_calibration(config: Dict[str, Any]) -> Dict[str, Any]:}+#: The report-identity vocabulary, copied from `laya.evals._IDENTITY_KEYS`. The copy is deliberate:+#: `laya.evals` imports numpy at module scope, and this module reads a checkpoint directory and a+#: report JSON with nothing but `json` and `os` on its import path. That is also why+#: `tests/test_evidence.py` holds the copy against the original in both directions -- a key renamed+#: upstream would make every report read as identity-incomplete here, and a key added upstream would+#: read as complete, and neither would fail anything else.+_IDENTITY_KEYS = ("schema", "dataset_sha256", "questions_sha256")++#: Read as provenance, never required for the identity verdict, because `laya.evals` records+#: `laya_version` in `config` and deliberately does not compare it+#: (`laya/evals.py::EvalReport.comparable_to`): a patch release must not invalidate a baseline.+_PROVENANCE_KEYS = ("laya_version",)++#: The part of that vocabulary two artifacts can actually disagree on. `schema` tags the report, not+#: the checkpoint, so it is identity for a report and never a checkpoint <-> report conflict.+_IDENTITY_CONFLICT_KEYS = ("dataset_sha256", "questions_sha256")++def inspect_report(report: Dict[str, Any]) -> Dict[str, Any]:- """Evidence state of an eval report's identity fields (reuses EvalReport semantics)."""+ """Evidence state of an eval report's identity fields, on the vocabulary declared above."""config = report.get("config")if not isinstance(config, dict):return _entry(UNKNOWN, "report has no config block; identity unknown")fields = {}- for key in ("schema", "dataset_sha256", "questions_sha256", "laya_version"):+ for key in _IDENTITY_KEYS + _PROVENANCE_KEYS:value = config.get(key)if value is None and report.get(key) is not None:value = report.get(key)fields[key] = _entry(PRESENT, "recorded", {"value": value}) if value is not None else _entry(UNKNOWN, "not recorded in this report")- identity_keys = [fields["schema"], fields["dataset_sha256"], fields["questions_sha256"]]- overall = PRESENT if all(f["state"] == PRESENT for f in identity_keys) else UNKNOWN+ overall = PRESENT if all(fields[key]["state"] == PRESENT for key in _IDENTITY_KEYS) else UNKNOWNreturn {"state": overall, "fields": fields}-_IDENTITY_CONFLICT_KEYS = ("dataset_sha256", "questions_sha256")--def _field(config_like: Any, key: str) -> Any:if not isinstance(config_like, dict):return Nonediff --git a/tests/test_evidence.py b/tests/test_evidence.pyindex da9b6fbf..5c28f67f 100644--- a/tests/test_evidence.py+++ b/tests/test_evidence.py@@ -107,6 +107,33 @@ toplevel = {"schema": "laya-evals-report/1", "dataset_sha256": "a", "questions_s"config": {}}check("identity at top level still read", evidence.inspect_report(toplevel)["state"], evidence.PRESENT)+# ------------------------------------------------------ identity vocabulary pinned to laya.evals+# `laya/evidence.py` copies the report-identity vocabulary instead of importing `laya.evals` (numpy at+# module scope), and a copy is only safe while it is held against the original -- the same way+# `tests/test_conformal_abstention.py` holds `confidence.BUCKET_QTYPES` against `common.temp_bucket`.+# Unpinned, a key renamed upstream would make every report read as identity-incomplete here, and a key+# added upstream would read as identity-complete; both silently, in the verdict this module exists to+# give. These four checks fail in both directions, on either side of the copy.+from laya import evals # noqa: E402++check("the copied identity vocabulary is evals' own, in order",+ list(evidence._IDENTITY_KEYS), list(evals._IDENTITY_KEYS))+check("the provenance key is recorded upstream but not compared there",+ [k for k in evidence._PROVENANCE_KEYS if k not in evals._IDENTITY_KEYS],+ list(evidence._PROVENANCE_KEYS))+check("checkpoint <-> report conflicts are the identity vocabulary minus the report tag",+ sorted(set(evals._IDENTITY_KEYS) - set(evidence._IDENTITY_CONFLICT_KEYS)), ["schema"])+check("inspect_report reads exactly its declared vocabulary",+ sorted(evidence.inspect_report({"config": {}})["fields"]),+ sorted(evidence._IDENTITY_KEYS + evidence._PROVENANCE_KEYS))++# ... and `schema` stays out of the conflict set on purpose: two artifacts that agree on the+# deterministic digests but carry different report tags are still the same measurement.+check("a differing schema is not a checkpoint <-> report conflict",+ evidence.relate({"dataset_sha256": "a"},+ {"config": {"dataset_sha256": "a", "schema": "laya-evals-report/2"}})["state"],+ evidence.UNKNOWN)+# ------------------------------------------------------ relationshipconfig = {"training": {"laya_train": {"laya_version": "0.3.28"}}}match = evidence.relate(config, {"config": {"laya_version": "0.3.28"}})