Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(evals): add opt-in per-slice quality gates

xpd9505319cb4a
diff --git a/docs/evals.md b/docs/evals.md
index 0e11aefc..9463888d 100644
--- a/docs/evals.md
+++ b/docs/evals.md
@@ -139,6 +139,45 @@ metrics per slice value, so a regression in one language or one question is visi
reading the aggregate. The `model` slice holds the checkpoint that answered each row: the
`Router`'s own choice per request, or the runner's `model` for a runner that does not route.
+### Opt-in slice gates
+
+The overall baseline gate can pass while a smaller language or question slice regresses. To make
+one reviewed slice a CI requirement, save a JSON policy such as `gates.json`:
+
+```json
+{
+ "version": 1,
+ "rules": [
+ {"slice": {"language": "zh"}, "metric": "choice_accuracy",
+ "min_count": 50, "max_drop": 0.05},
+ {"slice": {"qid": "intent"}, "metric": "ece",
+ "min_count": 50, "max": 0.10}
+ ]
+}
+```
+
+```bash
+laya-evals run data.jsonl --baseline baseline.json --tolerance choice_accuracy=0.02 \
+ --gate-policy gates.json --json report.json
+laya-evals compare report.json --baseline baseline.json \
+ --tolerance choice_accuracy=0.02 --gate-policy gates.json
+```
+
+Each rule selects exactly one `language`, `model`, `qid`, or `tag` value and names the metric
+exactly as it appears in the slice report. It has a positive `min_count` and exactly one limit:
+`min` or `max` checks the candidate value; `max_drop` permits at most that decrease from the
+baseline; `max_increase` permits at most that increase. The latter two require `--baseline`.
+The count is the number of scored answers for that metric in the selected slice, in **both**
+reports for a relative rule. For `ece`, it is the number of answers with a confidence/correctness
+pair. A missing slice or metric, too few scored answers, or skipped/errored cases fails the opted-in
+gate. Relative rules also require both reports to carry matching run identities, so missing
+evidence cannot appear as a pass. A measured regression reports the slice, metric, counts, values,
+and limit. Invalid policy syntax exits 2 before a checkpoint loads;
+a quality failure exits 1. The policy is recorded in `config.gate_policy` of a `run --json` report.
+
+The regular overall comparison still applies, including its tolerance and legacy-baseline
+behavior. Without `--gate-policy`, slice reporting and comparison behave as before.
+
## Run identity
`run` records what it measured in the report's `config` block, so the artifact a reviewer reads
diff --git a/laya/_eval_policy.py b/laya/_eval_policy.py
new file mode 100644
index 00000000..88eef954
--- /dev/null
+++ b/laya/_eval_policy.py
@@ -0,0 +1,169 @@
+"""Opt-in, weight-free quality gates over the slices already in an eval report."""
+
+import json
+import math
+from typing import Any, Dict, List, Optional
+
+from . import evals
+
+
+_DIMENSIONS = ("language", "model", "qid", "tag")
+_LIMITS = ("min", "max", "max_drop", "max_increase")
+_RELATIVE = ("max_drop", "max_increase")
+
+
+def _finite_number(value: Any) -> bool:
+ if type(value) not in (int, float):
+ return False
+ try:
+ return math.isfinite(value)
+ except OverflowError:
+ return False
+
+
+def load_policy(path: str) -> Dict[str, Any]:
+ """Read and validate a small, versioned JSON gate before a checkpoint is loaded."""
+ try:
+ with open(path, "r", encoding="utf-8") as handle:
+ policy = json.load(handle)
+ except (OSError, UnicodeError, json.JSONDecodeError) as exc:
+ raise evals.EvalError("invalid gate policy %r: %s" % (path, exc)) from exc
+ if not isinstance(policy, dict) or set(policy) != {"version", "rules"}:
+ raise evals.EvalError("gate policy needs exactly 'version' and 'rules'")
+ if type(policy["version"]) is not int or policy["version"] != 1:
+ raise evals.EvalError("gate policy version must be 1")
+ rules = policy["rules"]
+ if not isinstance(rules, list) or not rules:
+ raise evals.EvalError("gate policy 'rules' must be a non-empty list")
+ for index, rule in enumerate(rules, 1):
+ where = "gate policy rule %d" % index
+ if not isinstance(rule, dict):
+ raise evals.EvalError("%s must be an object" % where)
+ limits = set(rule) & set(_LIMITS)
+ if (set(rule) - {"slice", "metric", "min_count"} - set(_LIMITS)
+ or len(limits) != 1 or not {"slice", "metric", "min_count"} <= set(rule)):
+ raise evals.EvalError("%s needs slice, metric, min_count and exactly one limit" % where)
+ selector = rule["slice"]
+ if (not isinstance(selector, dict) or len(selector) != 1
+ or next(iter(selector)) not in _DIMENSIONS
+ or not isinstance(next(iter(selector.values())), str)
+ or not next(iter(selector.values())).strip()):
+ raise evals.EvalError("%s slice must name one non-empty language/model/qid/tag value" % where)
+ if not isinstance(rule["metric"], str) or not rule["metric"].strip():
+ raise evals.EvalError("%s metric must be a non-empty string" % where)
+ if type(rule["min_count"]) is not int or rule["min_count"] < 1:
+ raise evals.EvalError("%s min_count must be a positive integer" % where)
+ limit_name = next(iter(limits))
+ limit = rule[limit_name]
+ if (not _finite_number(limit)
+ or (limit_name in _RELATIVE and limit < 0)):
+ raise evals.EvalError("%s %s must be finite%s" %
+ (where, limit_name,
+ " and non-negative" if limit_name in _RELATIVE else ""))
+ return policy
+
+
+def needs_baseline(policy: Dict[str, Any]) -> bool:
+ return any(set(rule) & set(_RELATIVE) for rule in policy["rules"])
+
+
+def _count(report: evals.EvalReport, dimension: str, value: str, metric: str) -> int:
+ cases = report.cases if isinstance(report.cases, list) else []
+ count = 0
+ for case in cases:
+ if not isinstance(case, dict):
+ continue
+ if dimension == "tag":
+ tags = case.get("tags")
+ values = tags if isinstance(tags, (list, tuple)) else []
+ else:
+ values = [case.get(dimension)]
+ if value not in [str(item) for item in values if item is not None]:
+ continue
+ if metric == "ece":
+ confidence, correct = case.get("confidence"), case.get("correct")
+ valid = _finite_number(confidence) and isinstance(correct, bool)
+ else:
+ scores = case.get("scores")
+ score = scores.get(metric) if isinstance(scores, dict) else None
+ valid = _finite_number(score)
+ count += bool(valid)
+ return count
+
+
+def _value(report: evals.EvalReport, dimension: str, value: str,
+ metric: str) -> Optional[float]:
+ slices = report.slices if isinstance(report.slices, dict) else {}
+ dimension_slices = slices.get(dimension)
+ if not isinstance(dimension_slices, dict):
+ return None
+ group = dimension_slices.get(value)
+ if not isinstance(group, dict):
+ return None
+ result = group.get(metric)
+ if not _finite_number(result):
+ return None
+ return float(result)
+
+
+def check_policy(report: evals.EvalReport, policy: Dict[str, Any],
+ baseline: Optional[Dict[str, Any]] = None) -> List[str]:
+ """Return actionable failures. Missing measurements never count as a pass."""
+ failures: List[str] = []
+ base_report = None
+ if baseline is not None:
+ base_report = evals.EvalReport(
+ config=evals._identity_of(baseline),
+ slices=baseline.get("slices", {}), cases=baseline.get("cases", []))
+ if report.config.get("errored"):
+ failures.append("slice gate: candidate has skipped/errored cases")
+ if base_report is not None and base_report.config.get("errored"):
+ failures.append("slice gate: baseline has skipped/errored cases")
+ if needs_baseline(policy):
+ if base_report is None:
+ return failures + ["slice gate: relative rule needs a baseline"]
+ missing = [key for key in evals._IDENTITY_KEYS
+ if report.config.get(key) is None or base_report.config.get(key) is None]
+ if missing:
+ failures.append("slice gate: baseline and candidate need %s identity"
+ % ", ".join(missing))
+ comparable, reasons = report.comparable_to(baseline)
+ if not comparable:
+ failures.append("slice gate: baseline is not comparable: " + "; ".join(reasons))
+ for index, rule in enumerate(policy["rules"], 1):
+ dimension, value = next(iter(rule["slice"].items()))
+ metric = rule["metric"]
+ label = "slice gate rule %d (%s=%s, %s)" % (index, dimension, value, metric)
+ current = _value(report, dimension, value, metric)
+ current_count = _count(report, dimension, value, metric)
+ minimum = rule["min_count"]
+ if current is None:
+ failures.append("%s: metric or slice is missing" % label)
+ continue
+ if current_count < minimum:
+ failures.append("%s: candidate count %d is below min_count %d" %
+ (label, current_count, minimum))
+ continue
+ comparator = next(name for name in _LIMITS if name in rule)
+ limit = rule[comparator]
+ if comparator in _RELATIVE:
+ if base_report is None:
+ continue
+ previous = _value(base_report, dimension, value, metric)
+ previous_count = _count(base_report, dimension, value, metric)
+ if previous is None:
+ failures.append("%s: baseline metric or slice is missing" % label)
+ continue
+ if previous_count < minimum:
+ failures.append("%s: baseline count %d is below min_count %d" %
+ (label, previous_count, minimum))
+ continue
+ drift = previous - current if comparator == "max_drop" else current - previous
+ if drift > limit and not math.isclose(drift, limit, rel_tol=1e-12, abs_tol=1e-12):
+ failures.append("%s: candidate=%.4f (n=%d), baseline=%.4f (n=%d), %s=%.4f exceeded "
+ "by %.4f" % (label, current, current_count, previous,
+ previous_count, comparator, limit, drift - limit))
+ elif (comparator == "min" and current < limit) or (comparator == "max" and current > limit):
+ failures.append("%s: candidate=%.4f (n=%d), %s=%.4f failed" %
+ (label, current, current_count, comparator, limit))
+ return failures
diff --git a/laya/evals_cli.py b/laya/evals_cli.py
index a58375d0..95ccc47c 100644
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@ -17,7 +17,7 @@ import math
import sys
from typing import Any, Dict, List, Optional, Sequence, Tuple
-from . import evals
+from . import _eval_policy, evals
from .evals import EvalError
@@ -170,6 +170,8 @@ def _build_parser() -> argparse.ArgumentParser:
run.add_argument("--max", action="append", metavar="METRIC=VALUE", help="maximum for any metric")
run.add_argument("--slice", action="append", choices=("language", "model", "qid", "tag"),
help="also report this slice dimension; repeatable")
+ run.add_argument("--gate-policy", metavar="FILE",
+ help="apply opt-in per-slice quality rules from a JSON policy")
run.add_argument("--json", dest="json_out", help="write the full report JSON here")
run.add_argument("--markdown", dest="markdown_out", help="write a Markdown summary here")
@@ -178,6 +180,8 @@ def _build_parser() -> argparse.ArgumentParser:
compare.add_argument("--baseline", required=True)
compare.add_argument("--tolerance", action="append", metavar="METRIC=VALUE",
help="allowed absolute drift; repeatable")
+ compare.add_argument("--gate-policy", metavar="FILE",
+ help="apply opt-in per-slice quality rules from a JSON policy")
return parser
@@ -256,6 +260,10 @@ def _warn_no_value(extra: Sequence[evals.Evaluator], report: evals.EvalReport) -
def _cmd_run(args) -> int:
dataset = evals.Dataset.from_jsonl(args.dataset)
+ policy = _eval_policy.load_policy(args.gate_policy) if args.gate_policy else None
+ if policy and _eval_policy.needs_baseline(policy) and not args.baseline:
+ raise EvalError("--gate-policy has a relative rule; pass --baseline")
+ baseline = _load_report(args.baseline) if args.baseline and policy else None
if args.model:
for example in dataset.examples: # --model is authoritative over per-row model
example.model = args.model
@@ -295,6 +303,8 @@ def _cmd_run(args) -> int:
# refetched under the same name, and a reviewer comparing two reports needs the
# report -- not a file mtime -- to say so.
"dataset_sha256": evals.file_fingerprint(args.dataset)}
+ if policy:
+ config["gate_policy"] = policy
if args.onnx:
config["onnx"] = args.onnx
if extra:
@@ -328,7 +338,8 @@ def _cmd_run(args) -> int:
failures = _check_thresholds(report.overall, mins, maxs)
if args.baseline:
- baseline = _load_report(args.baseline)
+ if baseline is None:
+ baseline = _load_report(args.baseline)
ok, deltas = report.compare(baseline, tolerances)
_print_deltas(deltas)
if not ok:
@@ -338,6 +349,8 @@ def _cmd_run(args) -> int:
refusal = _comparability_failure(report, baseline)
if refusal:
failures.append(refusal)
+ if policy:
+ failures.extend(_eval_policy.check_policy(report, policy, baseline))
for name in sorted(report.overall):
print("%-18s %.4f" % (name, report.overall[name]))
@@ -368,6 +381,7 @@ def _cmd_compare(args) -> int:
# that disagreed with its baseline passed the gate while the same disagreement stated in
# `config` was refused. The baseline has always gone through `_identity_of`; this makes the
# candidate side symmetric.
+ policy = _eval_policy.load_policy(args.gate_policy) if args.gate_policy else None
document = _load_report(args.report)
report = evals.EvalReport(
config=evals._identity_of(document),
@@ -379,9 +393,12 @@ def _cmd_compare(args) -> int:
# comparability refusal has to apply here: a saved report that cannot say which experiment
# it came from must not pass a gate either.
refusal = _comparability_failure(report, baseline)
- if refusal:
- print("FAIL: " + refusal, file=sys.stderr)
- return 0 if ok and not refusal else 1
+ failures = [refusal] if refusal else []
+ if policy:
+ failures.extend(_eval_policy.check_policy(report, policy, baseline))
+ for failure in failures:
+ print("FAIL: " + failure, file=sys.stderr)
+ return 0 if ok and not failures else 1
def main(argv: Optional[Sequence[str]] = None) -> int:
diff --git a/tests/test_evals.py b/tests/test_evals.py
index e9e359a6..c6057211 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -1465,6 +1465,207 @@ def test_cli_accepts_a_checkpoint_alias_in_a_pin(tmp_path, monkeypatch):
assert recorded["revisions"] == {"english": SHA}
+# --------------------------------------------------------------- opt-in slice gates
+def _slice_gate_report(en_correct, zh_correct):
+ cases = [{"language": language, "scores": {"choice_accuracy": float(i < correct)},
+ "confidence": 0.8, "correct": i < correct}
+ for language, size, correct in (("en", 180, en_correct), ("zh", 20, zh_correct))
+ for i in range(size)]
+ return {"config": {"schema": evals.REPORT_SCHEMA, "dataset_sha256": "a" * 64,
+ "questions_sha256": "b" * 64},
+ "overall": {"choice_accuracy": (en_correct + zh_correct) / 200},
+ "slices": {"language": {"en": {"choice_accuracy": en_correct / 180},
+ "zh": {"choice_accuracy": zh_correct / 20}}},
+ "cases": cases}
+
+
+def _write_gate_policy(tmp_path, rule):
+ path = tmp_path / "gates.json"
+ path.write_text(json.dumps({"version": 1, "rules": [rule]}), encoding="utf-8")
+ return str(path)
+
+
+def test_cli_slice_gate_catches_regression_hidden_by_overall_gain(tmp_path, capsys):
+ from laya import evals_cli
+
+ baseline = tmp_path / "baseline.json"
+ candidate = tmp_path / "candidate.json"
+ baseline.write_text(json.dumps(_slice_gate_report(162, 18)), encoding="utf-8")
+ candidate.write_text(json.dumps(_slice_gate_report(171, 12)), encoding="utf-8")
+ policy = _write_gate_policy(tmp_path, {"slice": {"language": "zh"},
+ "metric": "choice_accuracy", "min_count": 20,
+ "max_drop": 0.05})
+ args = ["compare", str(candidate), "--baseline", str(baseline),
+ "--tolerance", "choice_accuracy=0.02"]
+ assert evals_cli.main(args) == 0, "90% -> 91.5% overall passes the existing gate"
+ capsys.readouterr()
+ assert evals_cli.main(args + ["--gate-policy", policy]) == 1
+ error = capsys.readouterr().err
+ assert "language=zh" in error and "candidate=0.6000 (n=20)" in error
+ assert "baseline=0.9000 (n=20)" in error and "max_drop=0.0500" in error
+
+
+def test_slice_gate_requires_scored_evidence_on_both_sides(tmp_path):
+ from laya import _eval_policy
+
+ policy = _eval_policy.load_policy(_write_gate_policy(
+ tmp_path, {"slice": {"language": "zh"}, "metric": "choice_accuracy",
+ "min_count": 20, "max_drop": 0.05}))
+ base = _slice_gate_report(162, 18)
+ candidate = _slice_gate_report(171, 12)
+ report = EvalReport(**candidate)
+ assert not any("count" in f for f in _eval_policy.check_policy(report, policy, base))
+ policy["rules"][0]["max_drop"] = 0.3
+ assert _eval_policy.check_policy(report, policy, base) == [], "the exact drift boundary passes"
+ policy["rules"][0]["max_drop"] = 0.05
+ report.cases = [c for c in report.cases if c["language"] != "zh"] + report.cases[-19:]
+ assert any("candidate count 19" in f for f in _eval_policy.check_policy(report, policy, base))
+ report = EvalReport(**candidate)
+ base["cases"] = [c for c in base["cases"] if c["language"] != "zh"] + base["cases"][-19:]
+ assert any("baseline count 19" in f for f in _eval_policy.check_policy(report, policy, base))
+ del base["slices"]["language"]["zh"]
+ assert any("baseline metric or slice is missing" in f
+ for f in _eval_policy.check_policy(report, policy, base))
+
+
+def test_slice_gate_absolute_and_increase_direction(tmp_path):
+ from laya import _eval_policy
+
+ candidate = _slice_gate_report(171, 12)
+ report = EvalReport(**candidate)
+ absolute = _eval_policy.load_policy(_write_gate_policy(
+ tmp_path, {"slice": {"language": "zh"}, "metric": "choice_accuracy",
+ "min_count": 20, "min": 0.7}))
+ assert any("min=0.7000 failed" in f for f in _eval_policy.check_policy(report, absolute))
+ candidate["slices"]["language"]["zh"].pop("choice_accuracy")
+ assert any("metric or slice is missing" in f for f in _eval_policy.check_policy(report, absolute))
+ candidate = _slice_gate_report(171, 12)
+ candidate["slices"]["language"]["zh"]["ece"] = 0.3
+ for case in candidate["cases"]:
+ if case["language"] == "zh":
+ case["confidence"] = 0.8
+ base = _slice_gate_report(162, 18)
+ base["slices"]["language"]["zh"]["ece"] = 0.1
+ increase = _eval_policy.load_policy(_write_gate_policy(
+ tmp_path, {"slice": {"language": "zh"}, "metric": "ece",
+ "min_count": 20, "max_increase": 0.05}))
+ assert any("max_increase=0.0500" in f
+ for f in _eval_policy.check_policy(EvalReport(**candidate), increase, base))
+
+
+def test_slice_gate_counts_each_tagged_answer_once_and_accepts_boundary(tmp_path):
+ from laya import _eval_policy
+
+ candidate = _slice_gate_report(171, 12)
+ for case in candidate["cases"]:
+ case.update(qid="intent", model="stub", tags=["critical", "critical"])
+ candidate["slices"].update({
+ "tag": {"critical": {"choice_accuracy": 183 / 200}},
+ "qid": {"intent": {"choice_accuracy": 183 / 200}},
+ "model": {"stub": {"choice_accuracy": 183 / 200}},
+ })
+ policy = _eval_policy.load_policy(_write_gate_policy(
+ tmp_path, {"slice": {"tag": "critical"}, "metric": "choice_accuracy",
+ "min_count": 200, "min": 0.915}))
+ assert _eval_policy.check_policy(EvalReport(**candidate), policy) == []
+ for dimension, value in (("qid", "intent"), ("model", "stub")):
+ policy["rules"][0]["slice"] = {dimension: value}
+ assert _eval_policy.check_policy(EvalReport(**candidate), policy) == []
+
+
+ {"slice": {"language": "zh"}, "metric": "choice_accuracy", "min_count": 20,
+ "min": 0.6, "max_drop": 0.1},
+ {"slice": {"language": "zh", "tag": "critical"}, "metric": "choice_accuracy",
+ "min_count": 20, "min": 0.6},
+ {"slice": {"language": "zh"}, "metric": "choice_accuracy", "min_count": 0,
+ "min": 0.6},
+ {"slice": {"language": "zh"}, "metric": "choice_accuracy", "min_count": 20,
+ "max_drop": -0.1},
+])
+def test_cli_rejects_malformed_slice_policy_before_loading(monkeypatch, tmp_path, rule):
+ from laya import evals_cli
+
+ built = _patch_router(monkeypatch)
+ dataset = _write_dataset(tmp_path, RUN_ROWS)
+ policy = _write_gate_policy(tmp_path, rule)
+ assert evals_cli.main(["run", dataset, "--gate-policy", policy]) == 2
+ assert not built
+
+
+def test_cli_relative_slice_rule_requires_baseline_before_loading(monkeypatch, tmp_path):
+ from laya import evals_cli
+
+ built = _patch_router(monkeypatch)
+ dataset = _write_dataset(tmp_path, RUN_ROWS)
+ policy = _write_gate_policy(tmp_path, {"slice": {"language": "zh"},
+ "metric": "choice_accuracy", "min_count": 1,
+ "max_drop": 0.1})
+ assert evals_cli.main(["run", dataset, "--gate-policy", policy]) == 2
+ assert not built
+
+
+def test_cli_rejects_invalid_policy_json_and_version(monkeypatch, tmp_path):
+ from laya import evals_cli
+
+ built = _patch_router(monkeypatch)
+ dataset = _write_dataset(tmp_path, RUN_ROWS)
+ path = tmp_path / "gates.json"
+ for content in ("{not json", json.dumps({"version": 2, "rules": [
+ {"slice": {"language": "zh"}, "metric": "choice_accuracy",
+ "min_count": 1, "min": 0.5}]})):
+ path.write_text(content, encoding="utf-8")
+ assert evals_cli.main(["run", dataset, "--gate-policy", str(path)]) == 2
+ assert not built
+
+
+def test_cli_run_slice_gate_and_saved_compare_share_policy(monkeypatch, tmp_path, capsys):
+ from laya import evals_cli
+
+ # Choice answers without confidence keep this gate-wiring test weight-free.
+ def answer(label):
+ return {"type": "choice", "choice": label}
+ rows = [{"state": language, "questions": Q, "expected": {"intent": "a"},
+ "language": language} for language in ("en", "zh")]
+ dataset = _write_dataset(tmp_path, rows)
+ baseline = tmp_path / "baseline.json"
+ candidate = tmp_path / "candidate.json"
+ _patch_router(monkeypatch, {"en": {"intent": answer("a")},
+ "zh": {"intent": answer("a")}})
+ assert evals_cli.main(["run", dataset, "--json", str(baseline)]) == 0
+ _patch_router(monkeypatch, {"en": {"intent": answer("a")},
+ "zh": {"intent": answer("b")}})
+ policy = _write_gate_policy(tmp_path, {"slice": {"language": "zh"},
+ "metric": "choice_accuracy", "min_count": 1,
+ "max_drop": 0.1})
+ flags = ["--baseline", str(baseline), "--tolerance", "choice_accuracy=1",
+ "--gate-policy", policy]
+ assert evals_cli.main(["run", dataset, *flags, "--json", str(candidate)]) == 1
+ assert json.loads(candidate.read_text())["config"]["gate_policy"]["rules"][0]["max_drop"] == 0.1
+ assert evals_cli.main(["compare", str(candidate), *flags]) == 1
+ assert "language=zh" in capsys.readouterr().err
+
+
+def test_slice_gate_fails_closed_on_identity_or_skipped_cases(tmp_path):
+ from laya import _eval_policy
+
+ candidate = _slice_gate_report(171, 12)
+ base = _slice_gate_report(162, 18)
+ policy = _eval_policy.load_policy(_write_gate_policy(
+ tmp_path, {"slice": {"language": "zh"}, "metric": "choice_accuracy",
+ "min_count": 20, "max_drop": 1.0}))
+ base["config"].pop("dataset_sha256")
+ assert any("need dataset_sha256 identity" in f
+ for f in _eval_policy.check_policy(EvalReport(**candidate), policy, base))
+ base["config"]["dataset_sha256"] = "c" * 64
+ assert any("not comparable" in f
+ for f in _eval_policy.check_policy(EvalReport(**candidate), policy, base))
+ base["config"]["dataset_sha256"] = "a" * 64
+ candidate["config"]["errored"] = [{"index": 1, "error": "missing answer"}]
+ assert any("skipped/errored" in f
+ for f in _eval_policy.check_policy(EvalReport(**candidate), policy, base))
+
+
@pytest.mark.parametrize("pairs, expected", [
(None, (None, {})),
([], (None, {})),