Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): read a report's identity wherever the report puts it
Aashishb1cb88a3eadc
diff --git a/laya/evidence.py b/laya/evidence.pyindex f88d4b8e..b30c5f8e 100644--- a/laya/evidence.py+++ b/laya/evidence.py@@ -7,8 +7,9 @@ train/eval path already persists:- a checkpoint directory's ``rl_agent_config.json`` (``training.laya_train`` and``training.laya_train_calibration`` as written by merged #931/#933);-- an existing ``laya-evals`` report JSON (its identity vocabulary is a copy of `laya.evals`' own,- pinned against it by `tests/test_evidence.py`).+- an existing ``laya-evals`` report JSON (its identity vocabulary and the rule for reading it --+ ``config`` first, the document itself as a fallback -- are a copy of `laya.evals`' own, pinned+ against it by `tests/test_evidence.py`).Every state it assigns is either derived directly from an upstream artifact ormarked UNKNOWN. It never manufactures a healthy-looking state from absent input.@@ -92,16 +93,36 @@ _PROVENANCE_KEYS = ("laya_version",)_IDENTITY_CONFLICT_KEYS = ("dataset_sha256", "questions_sha256")+def _report_identity(report: Dict[str, Any]) -> Dict[str, Any]:+ """The report's identity fields, wherever that report shape puts them.++ A report may state its identity in ``config`` or at the top level: `laya.evals._identity_of`+ reads ``config`` and falls back to the document for each key, and+ `laya/evals_cli.py::_cmd_compare` had to switch to it for exactly that reason. This module+ copies `laya.evals`' vocabulary rather than importing it (see `_IDENTITY_KEYS`), so it has to+ copy the read as well -- and copy it once. Read through ``config`` alone, the same document+ answers differently depending on which half of the toolchain parses it: an identity stated at+ the top level reads as absent, and a `dataset_sha256` that disagrees with the checkpoint reads+ as "no shared identity" instead of `INCOMPARABLE`. `tests/test_evidence.py` holds the copy+ against `_identity_of` in both directions, the way it holds `_IDENTITY_KEYS`.++ `_PROVENANCE_KEYS` ride along because `inspect_report` reports them; they are never compared,+ so this does not widen what `relate` can conflict on.+ """+ config = report.get("config")+ found = dict(config) if isinstance(config, dict) else {}+ for key in _IDENTITY_KEYS + _PROVENANCE_KEYS:+ if found.get(key) is None and report.get(key) is not None:+ found[key] = report[key]+ return found++def inspect_report(report: Dict[str, Any]) -> Dict[str, Any]:"""Evidence state of an eval report's identity fields, on the vocabulary declared above."""- config = report.get("config")- if not isinstance(config, dict):- return _entry(UNKNOWN, "report has no config block; identity unknown")+ identity = _report_identity(report)fields = {}for key in _IDENTITY_KEYS + _PROVENANCE_KEYS:- value = config.get(key)- if value is None and report.get(key) is not None:- value = report.get(key)+ value = identity.get(key)fields[key] = _entry(PRESENT, "recorded", {"value": value}) if value is not None else _entry(UNKNOWN, "not recorded in this report")overall = PRESENT if all(fields[key]["state"] == PRESENT for key in _IDENTITY_KEYS) else UNKNOWNreturn {"state": overall, "fields": fields}@@ -128,8 +149,12 @@ def relate(config: Dict[str, Any], report: Dict[str, Any]) -> Dict[str, Any]:infers a PRESENT match from `laya_version` (which names the software, not thecheckpoint). Everything else is UNKNOWN -- current artifacts expose no immutableshared checkpoint identity.++ "expose" is read through `_report_identity`, so a report that states its digests at the top+ level of the document rather than under ``config`` conflicts exactly like one that states them+ in ``config``. The disagreement itself is what the state claims, not where it was written."""- report_config = report.get("config") if isinstance(report.get("config"), dict) else {}+ report_config = _report_identity(report)conflicts = []for key in _IDENTITY_CONFLICT_KEYS:a = _field(config, key)diff --git a/tests/test_evidence.py b/tests/test_evidence.pyindex 887e6a65..de9dbeae 100644--- a/tests/test_evidence.py+++ b/tests/test_evidence.py@@ -107,13 +107,36 @@ toplevel = {"schema": "laya-evals-report/1", "dataset_sha256": "a", "questions_s"config": {}}check("identity at top level still read", evidence.inspect_report(toplevel)["state"], evidence.PRESENT)+# ... and with no `config` block at all, which is the shape that used to answer differently.+# `research/evals/act_head_eval.py` tags its document at the top level and writes no `config` key;+# bailing on the missing block made one document read as PRESENT once an empty `config` was added+# beside the same identity, and left a caller to branch on two different result shapes.+_noblock = {"schema": "laya-evals-report/1", "dataset_sha256": "a", "questions_sha256": "b"}+rep3 = evidence.inspect_report(_noblock)+check("identity at top level, no config block -> PRESENT", rep3["state"], evidence.PRESENT)+check("and it reads field for field like the same identity under config",+ rep3["fields"], evidence.inspect_report({"config": _noblock})["fields"])+check("no identity anywhere still reports its whole vocabulary",+ sorted(evidence.inspect_report({"overall": {}})["fields"]),+ sorted(evidence._IDENTITY_KEYS + evidence._PROVENANCE_KEYS))+# Provenance rides through the same read as identity (see `_report_identity`), so a document that+# records its version at the top level is not read as recording nothing.+check("provenance at the top level is read too",+ evidence.inspect_report({"config": {}, "laya_version": "0.3.28"})["fields"]+ ["laya_version"]["state"], evidence.PRESENT)+# ... by copying, not by writing into the caller's `config`: this inspection is read-only, and a+# view that aliased the report would leave the fallback values in the document it was handed.+_untouched = {"config": {}, "schema": "laya-evals-report/1"}+evidence.inspect_report(_untouched)+check("the report document is not mutated", _untouched, {"config": {}, "schema": "laya-evals-report/1"})+# ------------------------------------------------------ identity vocabulary pinned to laya.evals# `laya/evidence.py` copies the report-identity vocabulary instead of importing `laya.evals` (numpy at# module scope), and a copy is only safe while it is held against the original -- the same way# `tests/test_conformal_abstention.py` holds `confidence.BUCKET_QTYPES` against `common.temp_bucket`.# Unpinned, a key renamed upstream would make every report read as identity-incomplete here, and a key# added upstream would read as identity-complete; both silently, in the verdict this module exists to-# give. These four checks fail in both directions, on either side of the copy.+# give. These checks fail in both directions, on either side of the copy.from laya import evals # noqa: E402check("the copied identity vocabulary is evals' own, in order",@@ -127,12 +150,33 @@ check("inspect_report reads exactly its declared vocabulary",sorted(evidence.inspect_report({"config": {}})["fields"]),sorted(evidence._IDENTITY_KEYS + evidence._PROVENANCE_KEYS))+# The copy has to hold for the *read* as well, or the two halves of the toolchain disagree about+# which document says what: `_cmd_compare` only began refusing a candidate whose top-level `schema`+# differed from its baseline once it started reading through `evals._identity_of`. Same documents,+# same answer. `laya_version` is excluded because this module reports it and `evals` never looks+# for it.+_shadow = {"config": {"schema": "in config"}, "schema": "at the top level"}+for _doc in (_noblock, {"config": _noblock}, toplevel, full_report, {"overall": {}},+ {"config": {"dataset_sha256": "a"}, "schema": "laya-evals-report/1"}, _shadow):+ _here = {k: v for k, v in evidence._report_identity(_doc).items() if k in evidence._IDENTITY_KEYS}+ _there = {k: v for k, v in evals._identity_of(_doc).items() if k in evidence._IDENTITY_KEYS}+ check("the identity read is evals' own for %s" % json.dumps(_doc, sort_keys=True), _here, _there)+# Equality alone would still pass if both sides started preferring the document over `config`: the+# fallback is for a key `config` does not carry, not a way to shadow one it does.+check("config still outranks the top level", evidence._report_identity(_shadow)["schema"], "in config")+# ... and `schema` stays out of the conflict set on purpose: two artifacts that agree on the-# deterministic digests but carry different report tags are still the same measurement.+# deterministic digests but carry different report tags are still the same measurement. Both sides+# have to expose the tag for the check to say anything: with the checkpoint silent, the pair is+# skipped as "one side missing" whatever the conflict set holds.check("a differing schema is not a checkpoint <-> report conflict",evidence.relate({"dataset_sha256": "a"},{"config": {"dataset_sha256": "a", "schema": "laya-evals-report/2"}})["state"],evidence.UNKNOWN)+check("not even when both artifacts expose it",+ evidence.relate({"dataset_sha256": "a", "schema": "laya-evals-report/1"},+ {"config": {"dataset_sha256": "a", "schema": "laya-evals-report/2"}})["state"],+ evidence.UNKNOWN)# `docs/evals.md` carries the same vocabulary a third time -- it teaches that the identity fields# "are reported with the same semantics as `run`", and names the pair a conflict is claimed on -- so@@ -165,6 +209,18 @@ check("deterministic identity conflict -> INCOMPARABLE", conflict_ds["state"], eunknown = evidence.relate(config, {"config": {}})check("no shared identity field -> UNKNOWN", unknown["state"], evidence.UNKNOWN)+# The conflict is the disagreement, not the place it was written. `relate` used to read the report's+# `config` block only, so this document -- the shape `act_head_eval.py` writes -- stated a digest the+# checkpoint contradicts and still answered "no immutable shared checkpoint identity".+check("deterministic identity conflict at the top level -> INCOMPARABLE",+ evidence.relate({"dataset_sha256": "a"}, {"dataset_sha256": "b"})["state"], evidence.INCOMPARABLE)+check("top-level agreement is still never a claimed match",+ evidence.relate({"dataset_sha256": "a"}, {"dataset_sha256": "a"})["state"], evidence.UNKNOWN)+# Provenance rides through the same read because `inspect_report` reports it; it must not widen what+# `relate` can conflict on.+check("a differing laya_version at the top level is still not a conflict",+ evidence.relate({"laya_version": "0.3.28"}, {"laya_version": "0.3.20"})["state"], evidence.UNKNOWN)+# ------------------------------------------------------ CLI on a realistic artifact pairtmp = tempfile.mkdtemp()ckpt = os.path.join(tmp, "ckpt")@@ -184,6 +240,24 @@ check("evidence CLI returns 0 on realistic pair", rc, 0)rc_bad = evals_cli.main(["evidence", "--checkpoint", os.path.join(tmp, "nope")])check("missing checkpoint dir -> usage error 2", rc_bad, 2)+# ... and on the artifact the research harness actually writes: identity at the document's top+# level, no `config` key. Read through `inspect_checkpoint`/`format_summary`, which is the whole of+# what `evidence` prints, so this covers the summary's own shape as well as the verdicts.+_digest_ckpt = os.path.join(tmp, "ckpt-digests")+os.makedirs(_digest_ckpt)+with open(os.path.join(_digest_ckpt, "rl_agent_config.json"), "w", encoding="utf-8") as f:+ json.dump({"training": {"laya_train": {"dataset_sha256": "a"}}}, f)+_toplevel_report = os.path.join(tmp, "act-head-style.json")+with open(_toplevel_report, "w", encoding="utf-8") as f:+ json.dump({"schema": "laya-evals-report/1", "dataset_sha256": "b", "questions_sha256": "c"}, f)+_summary = evidence.format_summary(evidence.inspect_checkpoint(_digest_ckpt, _toplevel_report))+check_true("a top-level report identity reaches the summary as PRESENT",+ "eval report identity: PRESENT" in _summary)+check_true("and its differing digest reaches it as the conflict it names",+ "checkpoint<->report: INCOMPARABLE" in _summary)+check_true("every identity field is printed for a report with no config block",+ all((" %-18s" % key) in _summary for key in evidence._IDENTITY_KEYS))+# ------------------------------------------------------ no torch requiredimport subprocess_BLOCK = (