Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): the run identity was narrower than the rule it justifies itself with
Bruce-Yii6bbb338291f5
diff --git a/laya/evals.py b/laya/evals.pyindex 4e3b133d..27c7c57f 100644--- a/laya/evals.py+++ b/laya/evals.py@@ -126,7 +126,13 @@ class Dataset:def _canonical(value: Any) -> str:- return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False,+ # `sort_keys=False` on purpose. `examples/hooks/cache.py:22-25` states the rule this mirrors:+ # "Deliberately not `sort_keys=True`: a choice question's criteria order is positional, so two+ # orders are two questions, and `_question_schema` in `laya/router.py` keeps them apart for the+ # same reason." `sort_keys` reorders *dict* keys and leaves *lists* alone, so with a list-valued+ # `criteria` -- the shape a dataset row actually has -- this was already order-preserving; the+ # flag only ever mattered for a dict-valued one, where folding is the wrong direction.+ return json.dumps(value, sort_keys=False, separators=(",", ":"), ensure_ascii=False,default=str)@@ -170,6 +176,12 @@ def questions_fingerprint(dataset: "Dataset") -> str:rewrites. That is deliberate and asymmetric: two `noul` questions that render alike but arewritten differently would then be refused, which is the safe direction to be wrong in.Normalizing them would widen the change past what the identity needs.++ `labels` is hashed for the same reason `instructions` is: it is validated+ (`laya/agent.py:722-726` -> `_resolve_noul_labels`) and carried into the internal question+ (`laya/agent.py:748-749`), and `_resolve_noul_labels` (`laya/common.py:92`) turns it into the+ option text the model reads. Leaving it out let two question sets with different rendered+ options hash alike, which is the one thing this function exists to prevent."""schemas = set()for example in dataset.examples:@@ -177,6 +189,7 @@ def questions_fingerprint(dataset: "Dataset") -> str:body = question if isinstance(question, dict) else {}schemas.add(_canonical({"qid": qid, "type": body.get("type"),"instructions": _as_tokenized_instructions(body),+ "labels": body.get("labels"),"criteria": body.get("criteria")}))digest = hashlib.sha256()for schema in sorted(schemas):diff --git a/laya/evals_cli.py b/laya/evals_cli.pyindex e2ed02af..044aebc2 100644--- a/laya/evals_cli.py+++ b/laya/evals_cli.py@@ -348,8 +348,17 @@ def _cmd_run(args) -> int:def _cmd_compare(args) -> int:- report = evals.EvalReport(**{k: v for k, v in _load_report(args.report).items()- if k in ("config", "overall", "slices", "cases")})+ # `_identity_of` rather than a bare `config` slice, because a report may carry its identity+ # at the top level -- `research/evals/act_head_eval.py` puts `schema` there -- and+ # `EvalReport.comparable_to` reads the candidate's identity out of `config`. Filtering to the+ # four known keys dropped a top-level `schema` before the report was built, so a candidate+ # that disagreed with its baseline passed the gate while the same disagreement stated in+ # `config` was refused. The baseline has always gone through `_identity_of`; this makes the+ # candidate side symmetric.+ document = _load_report(args.report)+ report = evals.EvalReport(+ config=evals._identity_of(document),+ **{k: v for k, v in document.items() if k in ("overall", "slices", "cases")})baseline = _load_report(args.baseline)ok, deltas = report.compare(baseline, _parse_pairs(args.tolerance))_print_deltas(deltas)diff --git a/tests/test_evals.py b/tests/test_evals.pyindex d872259d..807a4b1c 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -970,6 +970,84 @@ def test_cli_compare_refuses_two_different_runs(tmp_path, capsys):assert "diff=" in captured.out and "choice_accuracy" in captured.out+def test_questions_fingerprint_covers_labels():+ """`labels` decides the option text, so it belongs in the key.++ The same argument `test_questions_fingerprint_covers_the_instructions` makes for+ `instructions`. `labels` is validated (`laya/agent.py:722-726` -> `_resolve_noul_labels`),+ carried into the internal question (`laya/agent.py:748-749`), and resolved into the option+ text the model actually reads (`laya/common.py:92`). Two question sets differing only in+ `labels` therefore ask the model different things, and hashing them alike let the gate pass+ a comparison it exists to refuse.+ """+ def with_labels(labels):+ q = {"verdict": {"type": "noul", "instructions": "Is this a refund?"}}+ if labels is not None:+ q["verdict"]["labels"] = labels+ return Dataset([Example("s", q, {"verdict": "true"})])++ default = evals.questions_fingerprint(with_labels(None))+ custom = evals.questions_fingerprint(with_labels(+ {"false": "denied: no money was requested",+ "true": "approved: the customer asked for money back"}))+ assert custom != default, "a change in `labels` left the fingerprint unchanged"+ # And the change is real: these are not two spellings of one thing.+ from laya.common import _resolve_noul_labels+ assert _resolve_noul_labels() != _resolve_noul_labels(+ {"false": "denied: no money was requested",+ "true": "approved: the customer asked for money back"})+++def test_questions_fingerprint_keeps_criteria_order():+ """A choice question's criteria order is positional, so two orders are two questions.++ `examples/hooks/cache.py:22-25` states the rule and `Router._question_schema`+ (`laya/router.py:141`) applies it with `sort_keys=False`. Worth pinning here for the dict+ form: `json.dumps(sort_keys=True)` reorders *dict keys* and leaves *lists* alone, so a+ list-valued `criteria` -- the shape a dataset row has -- was already safe. The flag only+ mattered for a dict-valued one, where folding is the wrong direction to be wrong in.+ """+ def with_criteria(criteria):+ q = {"pick": {"type": "choice", "instructions": "Pick one", "criteria": criteria}}+ return Dataset([Example("s", q, {"pick": "a"})])++ assert evals.questions_fingerprint(with_criteria(["alpha", "beta"])) != \+ evals.questions_fingerprint(with_criteria(["beta", "alpha"]))+ assert evals.questions_fingerprint(with_criteria({"a": "alpha", "b": "beta"})) != \+ evals.questions_fingerprint(with_criteria({"b": "beta", "a": "alpha"}))+++def test_a_top_level_schema_on_the_candidate_still_refuses(tmp_path, capsys):+ """The identity fallback is two-sided.++ `_identity_of` reads a report's identity from `config` *or* the top level, and its own comment+ says why: `research/evals/act_head_eval.py` puts `schema` there. But `comparable_to` applied+ it to the baseline only -- the candidate's identity came from `self.config`, and+ `laya/evals_cli.py` had already dropped a top-level `schema` when it built the `EvalReport`.+ So the *same* disagreement was refused when the candidate stated it in `config` and passed+ when it stated it at the top level.+ """+ from laya import evals_cli++ def write(name, doc):+ (tmp_path / name).write_text(json.dumps(doc), encoding="utf-8")+ return str(tmp_path / name)++ shared = {"overall": {"choice_accuracy": 0.8}, "cases": [], "slices": {}}+ baseline = write("b.json", dict(shared, schema="act-head-eval/2"))++ # Same mismatch, candidate states it in `config` -- refused.+ assert evals_cli.main(["compare", write("in_config.json", dict(+ shared, config={"schema": "act-head-eval/1"})), "--baseline", baseline]) == 1+ capsys.readouterr()++ # Candidate states it at the top level -- must refuse too, not pass.+ assert evals_cli.main(["compare", write("top_level.json", dict(+ shared, schema="act-head-eval/1")), "--baseline", baseline]) == 1+ err = capsys.readouterr().err+ assert "schema" in err++# ------------------------------------------------------------------ CLI run# One score row inside 0.25 of its label, one 0.3 away (inside 0.5 but not 0.25), plus a choice and# an noul row so every default metric is in the report too.