Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): clarify slice gate policy provenance
xp136ae2a54796
diff --git a/docs/evals.md b/docs/evals.mdindex 9463888d..56d1e5b2 100644--- a/docs/evals.md+++ b/docs/evals.md@@ -168,12 +168,15 @@ exactly as it appears in the slice report. It has a positive `min_count` and exa`min` or `max` checks the candidate value; `max_drop` permits at most that decrease from thebaseline; `max_increase` permits at most that increase. The latter two require `--baseline`.The count is the number of scored answers for that metric in the selected slice, in **both**-reports for a relative rule. For `ece`, it is the number of answers with a confidence/correctness-pair. A missing slice or metric, too few scored answers, or skipped/errored cases fails the opted-in-gate. Relative rules also require both reports to carry matching run identities, so missing-evidence cannot appear as a pass. A measured regression reports the slice, metric, counts, values,-and limit. Invalid policy syntax exits 2 before a checkpoint loads;-a quality failure exits 1. The policy is recorded in `config.gate_policy` of a `run --json` report.+reports for a relative rule. For `ece`, it is the number of answers with a finite confidence and+boolean `correct` value. A missing slice or metric, too few scored answers, or skipped/errored+cases fails the opted-in gate. Relative rules also require both reports to carry matching run+identities, so missing evidence cannot appear as a pass. A measured regression reports the slice,+metric, counts, values, and limit. Invalid policy syntax exits 2 before a checkpoint loads; a+quality failure exits 1. The policy is recorded in `config.gate_policy` of a `run --json` report.+`compare --gate-policy` applies the policy supplied on that command line to the saved measurements.+If it differs from the report's recorded policy, `compare` says so; an explicit re-check under a+new policy does not change the policy under which the original run was made.The regular overall comparison still applies, including its tolerance and legacy-baselinebehavior. Without `--gate-policy`, slice reporting and comparison behave as before.@@ -191,6 +194,7 @@ is reviewable on its own:| `questions_sha256` | a fingerprint of the question schema: every question's id, type, `instructions` and `criteria`, over the whole dataset || `laya_version` | the `laya` that computed the numbers || `thresholds` | the gate this run applied: `min`, `max` and `baseline_tolerance` |+| `gate_policy` | the optional slice gate policy applied by `run --gate-policy` || `revisions` | the commit each checkpoint that answered was loaded from (see [below](#baseline-and-ci-gate)) |`dataset` is a path, and a path is not an identity: a dataset can be edited in place, moved, ordiff --git a/laya/_eval_policy.py b/laya/_eval_policy.pyindex 88eef954..ed8bb4d6 100644--- a/laya/_eval_policy.py+++ b/laya/_eval_policy.py@@ -147,8 +147,6 @@ def check_policy(report: evals.EvalReport, policy: Dict[str, Any],comparator = next(name for name in _LIMITS if name in rule)limit = rule[comparator]if comparator in _RELATIVE:- if base_report is None:- continueprevious = _value(base_report, dimension, value, metric)previous_count = _count(base_report, dimension, value, metric)if previous is None:diff --git a/laya/evals_cli.py b/laya/evals_cli.pyindex 95ccc47c..dcb9d33a 100644--- a/laya/evals_cli.py+++ b/laya/evals_cli.py@@ -386,6 +386,9 @@ def _cmd_compare(args) -> int:report = evals.EvalReport(config=evals._identity_of(document),**{k: v for k, v in document.items() if k in ("overall", "slices", "cases")})+ if policy and report.config.get("gate_policy") not in (None, policy):+ print("laya-evals: warning: --gate-policy differs from the report's recorded gate_policy; "+ "this verdict uses --gate-policy", file=sys.stderr)baseline = _load_report(args.baseline)ok, deltas = report.compare(baseline, _parse_pairs(args.tolerance))_print_deltas(deltas)diff --git a/tests/test_evals.py b/tests/test_evals.pyindex c6057211..0409f852 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -943,7 +943,7 @@ def test_docs_document_the_run_identity_and_the_refusal():if key != "timing":assert "`%s`" % key in page, key# The keys the CLI adds, and the programmatic entry points, are part of the same contract.- for key in ("dataset_sha256", "thresholds", "revisions"):+ for key in ("dataset_sha256", "thresholds", "gate_policy", "revisions"):assert "`%s`" % key in page, keyfor name in ("REPORT_SCHEMA", "questions_fingerprint", "file_fingerprint", "comparable_to"):assert name in page, name@@ -1553,6 +1553,25 @@ def test_slice_gate_absolute_and_increase_direction(tmp_path):for f in _eval_policy.check_policy(EvalReport(**candidate), increase, base))+def test_slice_gate_ece_requires_boolean_correct_evidence(tmp_path):+ from laya import _eval_policy++ candidate = _slice_gate_report(171, 12)+ candidate["slices"]["language"]["zh"]["ece"] = 0.2+ for case in candidate["cases"]:+ if case["language"] == "zh":+ case["correct"] = int(case["correct"])+ policy = _eval_policy.load_policy(_write_gate_policy(+ tmp_path, {"slice": {"language": "zh"}, "metric": "ece",+ "min_count": 20, "max": 0.3}))+ assert any("candidate count 0 is below min_count 20" in failure+ for failure in _eval_policy.check_policy(EvalReport(**candidate), policy))+ for case in candidate["cases"]:+ if case["language"] == "zh":+ case["correct"] = bool(case["correct"])+ assert _eval_policy.check_policy(EvalReport(**candidate), policy) == []++def test_slice_gate_counts_each_tagged_answer_once_and_accepts_boundary(tmp_path):from laya import _eval_policy@@ -1567,10 +1586,15 @@ def test_slice_gate_counts_each_tagged_answer_once_and_accepts_boundary(tmp_pathpolicy = _eval_policy.load_policy(_write_gate_policy(tmp_path, {"slice": {"tag": "critical"}, "metric": "choice_accuracy","min_count": 200, "min": 0.915}))- assert _eval_policy.check_policy(EvalReport(**candidate), policy) == []+ report = EvalReport(**candidate)+ policy["rules"][0]["min_count"] = 201+ assert any("candidate count 200 is below min_count 201" in failure+ for failure in _eval_policy.check_policy(report, policy))+ policy["rules"][0]["min_count"] = 200+ assert _eval_policy.check_policy(report, policy) == []for dimension, value in (("qid", "intent"), ("model", "stub")):policy["rules"][0]["slice"] = {dimension: value}- assert _eval_policy.check_policy(EvalReport(**candidate), policy) == []+ assert _eval_policy.check_policy(report, policy) == []@pytest.mark.parametrize("rule", [@@ -1578,6 +1602,8 @@ def test_slice_gate_counts_each_tagged_answer_once_and_accepts_boundary(tmp_path"min": 0.6, "max_drop": 0.1},{"slice": {"language": "zh", "tag": "critical"}, "metric": "choice_accuracy","min_count": 20, "min": 0.6},+ {"slice": {"region": "cn"}, "metric": "choice_accuracy", "min_count": 20,+ "min": 0.6},{"slice": {"language": "zh"}, "metric": "choice_accuracy", "min_count": 0,"min": 0.6},{"slice": {"language": "zh"}, "metric": "choice_accuracy", "min_count": 20,@@ -1646,6 +1672,34 @@ def test_cli_run_slice_gate_and_saved_compare_share_policy(monkeypatch, tmp_pathassert "language=zh" in capsys.readouterr().err+def test_cli_compare_reports_a_different_recorded_gate_policy(tmp_path, capsys):+ from laya import evals_cli++ baseline = tmp_path / "baseline.json"+ candidate = tmp_path / "candidate.json"+ policy_path = tmp_path / "policy.json"+ strict = {"version": 1, "rules": [{"slice": {"language": "zh"},+ "metric": "choice_accuracy", "min_count": 20,+ "min": 0.99}]}+ lax = {"version": 1, "rules": [{"slice": {"language": "zh"},+ "metric": "choice_accuracy", "min_count": 20,+ "min": 0.0}]}+ baseline.write_text(json.dumps(_slice_gate_report(162, 18)), encoding="utf-8")+ document = _slice_gate_report(171, 12)+ document["config"]["gate_policy"] = strict+ candidate.write_text(json.dumps(document), encoding="utf-8")+ args = ["compare", str(candidate), "--baseline", str(baseline),+ "--tolerance", "choice_accuracy=0.02"]+ policy_path.write_text(json.dumps(lax), encoding="utf-8")+ assert evals_cli.main(args + ["--gate-policy", str(policy_path)]) == 0+ assert "warning: --gate-policy differs from the report's recorded gate_policy" in capsys.readouterr().err+ policy_path.write_text(json.dumps(strict), encoding="utf-8")+ assert evals_cli.main(args + ["--gate-policy", str(policy_path)]) == 1+ assert "warning: --gate-policy differs" not in capsys.readouterr().err+ assert evals_cli.main(args) == 0+ assert capsys.readouterr().err == ""++def test_slice_gate_fails_closed_on_identity_or_skipped_cases(tmp_path):from laya import _eval_policy