Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): clarify slice gate policy provenance

xp136ae2a54796
diff --git a/docs/evals.md b/docs/evals.md
index 9463888d..56d1e5b2 100644
--- a/docs/evals.md
+++ b/docs/evals.md
@@ -168,12 +168,15 @@ exactly as it appears in the slice report. It has a positive `min_count` and exa
`min` or `max` checks the candidate value; `max_drop` permits at most that decrease from the
baseline; `max_increase` permits at most that increase. The latter two require `--baseline`.
The count is the number of scored answers for that metric in the selected slice, in **both**
-reports for a relative rule. For `ece`, it is the number of answers with a confidence/correctness
-pair. A missing slice or metric, too few scored answers, or skipped/errored cases fails the opted-in
-gate. Relative rules also require both reports to carry matching run identities, so missing
-evidence cannot appear as a pass. A measured regression reports the slice, metric, counts, values,
-and limit. Invalid policy syntax exits 2 before a checkpoint loads;
-a quality failure exits 1. The policy is recorded in `config.gate_policy` of a `run --json` report.
+reports for a relative rule. For `ece`, it is the number of answers with a finite confidence and
+boolean `correct` value. A missing slice or metric, too few scored answers, or skipped/errored
+cases fails the opted-in gate. Relative rules also require both reports to carry matching run
+identities, so missing evidence cannot appear as a pass. A measured regression reports the slice,
+metric, counts, values, and limit. Invalid policy syntax exits 2 before a checkpoint loads; a
+quality failure exits 1. The policy is recorded in `config.gate_policy` of a `run --json` report.
+`compare --gate-policy` applies the policy supplied on that command line to the saved measurements.
+If it differs from the report's recorded policy, `compare` says so; an explicit re-check under a
+new policy does not change the policy under which the original run was made.
The regular overall comparison still applies, including its tolerance and legacy-baseline
behavior. Without `--gate-policy`, slice reporting and comparison behave as before.
@@ -191,6 +194,7 @@ is reviewable on its own:
| `questions_sha256` | a fingerprint of the question schema: every question's id, type, `instructions` and `criteria`, over the whole dataset |
| `laya_version` | the `laya` that computed the numbers |
| `thresholds` | the gate this run applied: `min`, `max` and `baseline_tolerance` |
+| `gate_policy` | the optional slice gate policy applied by `run --gate-policy` |
| `revisions` | the commit each checkpoint that answered was loaded from (see [below](#baseline-and-ci-gate)) |
`dataset` is a path, and a path is not an identity: a dataset can be edited in place, moved, or
diff --git a/laya/_eval_policy.py b/laya/_eval_policy.py
index 88eef954..ed8bb4d6 100644
--- a/laya/_eval_policy.py
+++ b/laya/_eval_policy.py
@@ -147,8 +147,6 @@ def check_policy(report: evals.EvalReport, policy: Dict[str, Any],
comparator = next(name for name in _LIMITS if name in rule)
limit = rule[comparator]
if comparator in _RELATIVE:
- if base_report is None:
- continue
previous = _value(base_report, dimension, value, metric)
previous_count = _count(base_report, dimension, value, metric)
if previous is None:
diff --git a/laya/evals_cli.py b/laya/evals_cli.py
index 95ccc47c..dcb9d33a 100644
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@ -386,6 +386,9 @@ def _cmd_compare(args) -> int:
report = evals.EvalReport(
config=evals._identity_of(document),
**{k: v for k, v in document.items() if k in ("overall", "slices", "cases")})
+ if policy and report.config.get("gate_policy") not in (None, policy):
+ print("laya-evals: warning: --gate-policy differs from the report's recorded gate_policy; "
+ "this verdict uses --gate-policy", file=sys.stderr)
baseline = _load_report(args.baseline)
ok, deltas = report.compare(baseline, _parse_pairs(args.tolerance))
_print_deltas(deltas)
diff --git a/tests/test_evals.py b/tests/test_evals.py
index c6057211..0409f852 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -943,7 +943,7 @@ def test_docs_document_the_run_identity_and_the_refusal():
if key != "timing":
assert "`%s`" % key in page, key
# The keys the CLI adds, and the programmatic entry points, are part of the same contract.
- for key in ("dataset_sha256", "thresholds", "revisions"):
+ for key in ("dataset_sha256", "thresholds", "gate_policy", "revisions"):
assert "`%s`" % key in page, key
for name in ("REPORT_SCHEMA", "questions_fingerprint", "file_fingerprint", "comparable_to"):
assert name in page, name
@@ -1553,6 +1553,25 @@ def test_slice_gate_absolute_and_increase_direction(tmp_path):
for f in _eval_policy.check_policy(EvalReport(**candidate), increase, base))
+def test_slice_gate_ece_requires_boolean_correct_evidence(tmp_path):
+ from laya import _eval_policy
+
+ candidate = _slice_gate_report(171, 12)
+ candidate["slices"]["language"]["zh"]["ece"] = 0.2
+ for case in candidate["cases"]:
+ if case["language"] == "zh":
+ case["correct"] = int(case["correct"])
+ policy = _eval_policy.load_policy(_write_gate_policy(
+ tmp_path, {"slice": {"language": "zh"}, "metric": "ece",
+ "min_count": 20, "max": 0.3}))
+ assert any("candidate count 0 is below min_count 20" in failure
+ for failure in _eval_policy.check_policy(EvalReport(**candidate), policy))
+ for case in candidate["cases"]:
+ if case["language"] == "zh":
+ case["correct"] = bool(case["correct"])
+ assert _eval_policy.check_policy(EvalReport(**candidate), policy) == []
+
+
def test_slice_gate_counts_each_tagged_answer_once_and_accepts_boundary(tmp_path):
from laya import _eval_policy
@@ -1567,10 +1586,15 @@ def test_slice_gate_counts_each_tagged_answer_once_and_accepts_boundary(tmp_path
policy = _eval_policy.load_policy(_write_gate_policy(
tmp_path, {"slice": {"tag": "critical"}, "metric": "choice_accuracy",
"min_count": 200, "min": 0.915}))
- assert _eval_policy.check_policy(EvalReport(**candidate), policy) == []
+ report = EvalReport(**candidate)
+ policy["rules"][0]["min_count"] = 201
+ assert any("candidate count 200 is below min_count 201" in failure
+ for failure in _eval_policy.check_policy(report, policy))
+ policy["rules"][0]["min_count"] = 200
+ assert _eval_policy.check_policy(report, policy) == []
for dimension, value in (("qid", "intent"), ("model", "stub")):
policy["rules"][0]["slice"] = {dimension: value}
- assert _eval_policy.check_policy(EvalReport(**candidate), policy) == []
+ assert _eval_policy.check_policy(report, policy) == []
@pytest.mark.parametrize("rule", [
@@ -1578,6 +1602,8 @@ def test_slice_gate_counts_each_tagged_answer_once_and_accepts_boundary(tmp_path
"min": 0.6, "max_drop": 0.1},
{"slice": {"language": "zh", "tag": "critical"}, "metric": "choice_accuracy",
"min_count": 20, "min": 0.6},
+ {"slice": {"region": "cn"}, "metric": "choice_accuracy", "min_count": 20,
+ "min": 0.6},
{"slice": {"language": "zh"}, "metric": "choice_accuracy", "min_count": 0,
"min": 0.6},
{"slice": {"language": "zh"}, "metric": "choice_accuracy", "min_count": 20,
@@ -1646,6 +1672,34 @@ def test_cli_run_slice_gate_and_saved_compare_share_policy(monkeypatch, tmp_path
assert "language=zh" in capsys.readouterr().err
+def test_cli_compare_reports_a_different_recorded_gate_policy(tmp_path, capsys):
+ from laya import evals_cli
+
+ baseline = tmp_path / "baseline.json"
+ candidate = tmp_path / "candidate.json"
+ policy_path = tmp_path / "policy.json"
+ strict = {"version": 1, "rules": [{"slice": {"language": "zh"},
+ "metric": "choice_accuracy", "min_count": 20,
+ "min": 0.99}]}
+ lax = {"version": 1, "rules": [{"slice": {"language": "zh"},
+ "metric": "choice_accuracy", "min_count": 20,
+ "min": 0.0}]}
+ baseline.write_text(json.dumps(_slice_gate_report(162, 18)), encoding="utf-8")
+ document = _slice_gate_report(171, 12)
+ document["config"]["gate_policy"] = strict
+ candidate.write_text(json.dumps(document), encoding="utf-8")
+ args = ["compare", str(candidate), "--baseline", str(baseline),
+ "--tolerance", "choice_accuracy=0.02"]
+ policy_path.write_text(json.dumps(lax), encoding="utf-8")
+ assert evals_cli.main(args + ["--gate-policy", str(policy_path)]) == 0
+ assert "warning: --gate-policy differs from the report's recorded gate_policy" in capsys.readouterr().err
+ policy_path.write_text(json.dumps(strict), encoding="utf-8")
+ assert evals_cli.main(args + ["--gate-policy", str(policy_path)]) == 1
+ assert "warning: --gate-policy differs" not in capsys.readouterr().err
+ assert evals_cli.main(args) == 0
+ assert capsys.readouterr().err == ""
+
+
def test_slice_gate_fails_closed_on_identity_or_skipped_cases(tmp_path):
from laya import _eval_policy