Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(reporting): show judge-mixed lift exclusions on Tier 3 evaluator cards (#418)

Narendran Raghavand55b24033016
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 893be9c1..e8fda5ad 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -380,6 +380,18 @@ All notable changes to SkillEvaluator are documented in this file.
confirm a guess for the masked part. Redaction now removes a digest that no
longer matches the reported value.
+- Tier 3 HTML evaluator cards now say when a judge metric's lift left out cases
+ whose arms were scored by different judge models, for example after a judge
+ fallback in one arm. The arm scores keep those cases, so a card could show
+ Baseline 0.20 and With skill 0.50 next to a -0.40 lift with nothing
+ explaining the gap. The card now notes `Lift leaves out 1 case scored by
+ different judge models; the arm scores include it.`, or says that no
+ comparable result remains when the card has no lift, and labels those cases
+ `Left out of lift: different judge models` in its per-trial evidence. In the
+ canonical JSON the evaluator records the count in `lift_excluded_cases`, its
+ card lists the cases in `comparison_basis`, and the matching evidence entries
+ carry `comparison_excluded: true`. Scores and lift are unchanged.
+
- Tier 2 context deduplication now numbers `SHARED CONTENT n` blocks from the
most repeated text, ties in first-seen order, instead of in first-seen order
alone. In 0.8.0 a text repeated hundreds of times could get block 10 or later,
diff --git a/docs/reports.mdx b/docs/reports.mdx
index e9da4d19..db384028 100644
--- a/docs/reports.mdx
+++ b/docs/reports.mdx
@@ -884,7 +884,18 @@ cases, while evaluator, dimension, and overall lift use the eligible
comparisons. A reported lift can therefore differ from subtracting the two
displayed arm scores, even reversing their apparent direction. When no complete
comparison with matching judge models remains, the affected metric, dimension,
-and overall lift are unavailable rather than zero.
+and overall lift are unavailable rather than zero. The HTML report's evaluator
+card says how many cases its lift left out, or, when it has no lift, that no
+comparable result remains, and its per-trial evidence labels those cases
+**Left out of lift: different judge models**. In the canonical JSON the
+evaluator records that count in `lift_excluded_cases`. Its card's
+`comparison_basis` lists in `case_ids` the lift's keys for the cases the loaded
+rewards identify: each case ID without an `-attemptN` or `_attemptN` suffix,
+cut to 256 characters and listed once, at most 64 keys. It also gives the
+with-skill `excluded_attempts` and `contributing_attempts`, which are both
+`null` unless both arms' rewards loaded completely and identify every left-out
+case. The matching evidence entries carry `comparison_excluded: true`. A
+never-activated agent has no lift, so it gets none of these fields.
If every positive with-skill trial has confirmed zero activation, the agent's
result is **INCONCLUSIVE: skill never activated**. Reports keep its arm scores
diff --git a/src/skillevaluator/evaluation/tier3_report.py b/src/skillevaluator/evaluation/tier3_report.py
index caf0bcdc..6b386bd4 100644
--- a/src/skillevaluator/evaluation/tier3_report.py
+++ b/src/skillevaluator/evaluation/tier3_report.py
@@ -21,6 +21,7 @@ import contextlib
import json
import logging
import math
+from collections.abc import Collection
from dataclasses import dataclass, field
from itertools import islice
from pathlib import Path
@@ -108,6 +109,12 @@ _MAX_RAW_METRICS_PER_REWARD = 64
_MAX_RAW_REWARD_FIELDS = 96
_MAX_CUSTOM_METRIC_NAME_VISITS_PER_REWARD = 128
_MAX_UNPAIRED_CASE_IDS_IN_REPORT = 64
+# An evaluator card lists at most this many of the cases its lift left out; the
+# card's ``excluded_cases`` count stays exact.
+_MAX_LIFT_EXCLUDED_CASE_IDS_PER_CARD = 64
+_MAX_LIFT_EXCLUDED_CASE_ID_CHARS = 256
+# Lift pairing cannot compare a trial without a resolved case.
+_UNRESOLVED_CASE_IDS = frozenset({"", "unknown"})
_MAX_EMBEDDED_REPORT_BYTES = 2 * 1024 * 1024
_MAX_JSON_SAFE_INTEGER = (1 << 53) - 1
@@ -1856,6 +1863,77 @@ def _not_applicable_dimensions(
return out
+def _logical_condition_rewards(info: dict[str, Any]) -> tuple[list[list[dict[str, Any]]], bool]:
+ """Return each arm's logical attempts as the lift rebuild compares them, and whether both loads are complete.
+
+ The first list holds the with-skill attempts, the second the baseline's. A
+ logical trial whose rows disagree on the case is left out and makes the
+ result incomplete.
+ """
+ from skillevaluator.tier3.harbor.collector import _logical_attempt_rewards
+ from skillevaluator.tier3.harbor.report_data import logical_trial_reward_groups
+
+ conditions: list[list[dict[str, Any]]] = []
+ complete = True
+ for key in ("rewards", "rewards_baseline"):
+ rows = info.get(key)
+ complete = complete and info.get(f"{key}_complete") is True
+ groups = logical_trial_reward_groups(
+ [row for row in rows if isinstance(row, dict)] if isinstance(rows, list) else []
+ )
+ logical_rows: list[dict[str, Any]] = []
+ for index, group in enumerate(groups):
+ if not _logical_group_entry_identity_is_consistent(group):
+ complete = False
+ continue
+ logical_rows.extend(
+ {
+ **row,
+ "_trial_root_name": str(index),
+ **({"_step_name": str(step)} if len(group) > 1 else {}),
+ }
+ for step, row in enumerate(group)
+ )
+ conditions.append(_logical_attempt_rewards(logical_rows))
+ return conditions, complete
+
+
+def _judge_mixed_lift_exclusions(info: dict[str, Any], metrics: Collection[str]) -> dict[str, dict[str, Any]]:
+ """Name the cases each judge metric's lift left out because the arms' judge models differed.
+
+ Reads the logical attempts the lift rebuild compares, so a case listed here
+ is one that ``_compute_lift`` left out of that metric. Per metric it returns
+ ``case_ids``, ``excluded_attempts``, the with-skill attempts that scored the
+ metric on those cases, and ``complete``, whether both arms' rewards loaded
+ completely with every logical trial naming one case. A saved run without
+ loaded rewards yields no entry; its recorded per-metric counts still
+ describe the exclusion.
+ """
+ from skillevaluator.tier3.harbor.collector import _entry_id, _judge_mixed_cases
+ from skillevaluator.tier3.harbor.metrics import metric_set_for_reward, metric_value
+
+ (with_rows, without_rows), complete = _logical_condition_rewards(info)
+ if not with_rows or not without_rows:
+ return {}
+ exclusions: dict[str, dict[str, Any]] = {}
+ for metric, cases in _judge_mixed_cases(with_rows, without_rows, None).items():
+ case_ids = {case for case in cases if case not in _UNRESOLVED_CASE_IDS}
+ if metric not in metrics or not case_ids:
+ continue
+ exclusions[metric] = {
+ "case_ids": case_ids,
+ "excluded_attempts": sum(
+ 1
+ for row in with_rows
+ if _entry_id(row) in case_ids
+ and metric in metric_set_for_reward(row)[1]
+ and metric_value(row, metric) is not None
+ ),
+ "complete": complete,
+ }
+ return exclusions
+
+
def _judge_aware_lift(
info: dict[str, Any],
with_scores: dict[str, Any],
@@ -1866,10 +1944,8 @@ def _judge_aware_lift(
from skillevaluator.tier3.harbor.collector import (
_LLM_JUDGE_METRICS,
_compute_lift,
- _logical_attempt_rewards,
_reward_judge_models,
)
- from skillevaluator.tier3.harbor.report_data import logical_trial_reward_groups
recorded = info.get("lift")
recorded = recorded if isinstance(recorded, dict) else {}
@@ -1919,28 +1995,7 @@ def _judge_aware_lift(
)
):
return None
- conditions: list[list[dict[str, Any]]] = []
- complete = True
- for key in ("rewards", "rewards_baseline"):
- rows = info.get(key)
- complete = complete and info.get(f"{key}_complete") is True
- groups = logical_trial_reward_groups(
- [row for row in rows if isinstance(row, dict)] if isinstance(rows, list) else []
- )
- logical_rows: list[dict[str, Any]] = []
- for index, group in enumerate(groups):
- if not _logical_group_entry_identity_is_consistent(group):
- complete = False
- continue
- logical_rows.extend(
- {
- **row,
- "_trial_root_name": str(index),
- **({"_step_name": str(step)} if len(group) > 1 else {}),
- }
- for step, row in enumerate(group)
- )
- conditions.append(_logical_attempt_rewards(logical_rows))
+ conditions, complete = _logical_condition_rewards(info)
clean_scores = [
{metric: value for metric, raw in scores.items() if (value := _unit_interval_score(raw)) is not None}
for scores in (with_scores, without_scores)
@@ -2114,6 +2169,7 @@ def _build_agent(
if never_activated:
for evaluator in evaluators.values():
evaluator["lift"] = None
+ evaluator.pop("lift_excluded_cases", None)
for dimension in dimensions:
dimension.update(
lift=None,
@@ -2317,8 +2373,10 @@ def _attach_agent_report_details(
custom_with_skill = info.get("custom_with_skill")
custom_without_skill = info.get("custom_without_skill")
custom_lift = info.get("custom_lift")
+ evaluators = agent_payload.get("evaluators", {})
+ lift_excluded_metrics = [metric for metric, values in evaluators.items() if values.get("lift_excluded_cases")]
agent_payload["evaluator_cards"] = _evaluator_cards(
- agent_payload.get("evaluators", {}),
+ evaluators,
# Baseline-only cards draw their evidence from the only arm that ran.
rewards=(info.get("rewards_baseline") if baseline_only else info.get("rewards")) or [],
custom_with_skill=custom_with_skill if isinstance(custom_with_skill, dict) else {},
@@ -2326,6 +2384,8 @@ def _attach_agent_report_details(
custom_lift=custom_lift if isinstance(custom_lift, dict) else {},
report_budget=report_budget,
baseline_only=baseline_only,
+ lift_exclusions=_judge_mixed_lift_exclusions(info, lift_excluded_metrics) if lift_excluded_metrics else {},
+ lift_basis=agent_payload.get("lift_basis"),
)
if _never_activated(agent_payload):
for card in agent_payload["evaluator_cards"]:
@@ -2373,9 +2433,21 @@ def _build_evaluators(
else ("not_applicable" if metric in with_not_applicable else "missing"),
"baseline_status": baseline_status,
}
+ if excluded_cases := _lift_excluded_case_count(mixed, metric):
+ # The arm scores keep these cases; only this metric's lift left them out.
+ evaluators[metric]["lift_excluded_cases"] = excluded_cases
return evaluators
+def _lift_excluded_case_count(mixed: object, metric: str) -> int:
+ """Return how many cases a judge metric's lift left out because the arms' judge models differed."""
+ from skillevaluator.tier3.harbor.collector import _LLM_JUDGE_METRICS
+
+ count = mixed.get(metric) if isinstance(mixed, dict) and metric in _LLM_JUDGE_METRICS else None
+ # A count JSON cannot carry exactly is left out rather than published as ``null``.
+ return count if type(count) is int and 0 < count <= _MAX_JSON_SAFE_INTEGER else 0
+
+
def _build_dimensions(
with_scores: dict[str, Any],
without_scores: dict[str, Any],
@@ -2692,8 +2764,11 @@ def _metric_evidence(
rewards: list[dict[str, Any]],
report_budget: _ReportBudget,
sampling: dict[str, Any] | None = None,
+ *,
+ lift_excluded_case_ids: Collection[str] = (),
) -> list[dict[str, Any]]:
from skillevaluator.tier3.eval_core.atif_helpers import _METRIC_EVIDENCE_REF_METRICS
+ from skillevaluator.tier3.harbor.collector import _entry_id
from skillevaluator.tier3.harbor.metrics import DEFAULT_METRICS, metric_set_for_reward
if report_budget.evidence_remaining <= 0:
@@ -2755,6 +2830,10 @@ def _metric_evidence(
"checks": checks,
"evidence_refs": _compact_evidence_refs(detail.get("evidence_refs")),
}
+ if lift_excluded_case_ids and _entry_id(reward) in lift_excluded_case_ids:
+ # The case's arms were scored by different judge models: its score
+ # counts in the arm score but not in this metric's lift.
+ entry["comparison_excluded"] = True
if score_failed and metric in _METRIC_EVIDENCE_REF_METRICS and not entry["evidence_refs"]:
from skillevaluator.evaluation.tier3_findings import evidence_unavailable_reason
@@ -2922,6 +3001,8 @@ def _evaluator_card(
rewards: list[dict[str, Any]],
report_budget: _ReportBudget,
baseline_only: bool = False,
+ lift_exclusion: dict[str, Any] | None = None,
+ lift_basis: dict[str, Any] | None = None,
) -> dict[str, Any]:
if baseline_only:
# No with-skill arm ran: the baseline score sets the card's status band.
@@ -2929,6 +3010,10 @@ def _evaluator_card(
headline = _as_float(scores.get("baseline"))
else:
ws = headline = _as_float(scores.get("with_skill"))
+ excluded_cases = scores.get("lift_excluded_cases")
+ if baseline_only or type(excluded_cases) is not int or excluded_cases <= 0:
+ excluded_cases = 0
+ excluded_case_ids: set[str] = set((lift_exclusion or {}).get("case_ids") or ()) if excluded_cases else set()
evidence_sampling: dict[str, Any] = {}
card = {
"id": metric,
@@ -2937,13 +3022,68 @@ def _evaluator_card(
"baseline": scores.get("baseline"),
"lift": scores.get("lift"),
"status": "pass" if headline >= 0.8 else ("warn" if headline >= 0.6 else "fail"),
- "evidence": _metric_evidence(metric, rewards, report_budget, evidence_sampling),
+ "evidence": _metric_evidence(
+ metric, rewards, report_budget, evidence_sampling, lift_excluded_case_ids=excluded_case_ids
+ ),
}
if evidence_sampling:
card["evidence_sampling"] = evidence_sampling
+ if excluded_cases:
+ card["comparison_basis"] = _card_comparison_basis(
+ metric,
+ excluded_cases,
+ excluded_case_ids,
+ lift_exclusion,
+ lift_basis,
+ arm_scores=scores.get("with_skill") is not None and scores.get("baseline") is not None,
+ )
return card
+def _card_comparison_basis(
+ metric: str,
+ excluded_cases: int,
+ case_ids: set[str],
+ lift_exclusion: dict[str, Any] | None,
+ lift_basis: dict[str, Any] | None,
+ *,
+ arm_scores: bool,
+) -> dict[str, Any]:
+ """Explain which cases an evaluator card's lift left out while its arm scores keep them.
+
+ ``excluded_cases`` is the comparison's own count. ``case_ids`` lists the
+ lift's keys for the cases the loaded rewards identify (case IDs without an
+ attempt suffix), which can be fewer, for example for a saved run whose
+ rewards were not loaded. The with-skill attempt counts are both ``None``
+ unless both arms' rewards loaded completely and identify every left-out
+ case: ``excluded_attempts`` scored the metric on those cases, and
+ ``contributing_attempts`` fed the metric's lift. ``arm_scores`` says whether
+ both arm scores exist for the reason to point at.
+ """
+ exclusion = lift_exclusion or {}
+ # The lift basis is published whenever both arms' reward loads report complete,
+ # even when a logical trial named two cases and left the comparison short.
+ complete = exclusion.get("complete") is True and isinstance(lift_basis, dict) and len(case_ids) == excluded_cases
+ metrics_basis = lift_basis.get("metrics") if complete else None
+ metric_basis = metrics_basis.get(metric) if isinstance(metrics_basis, dict) else None
+ trials = metric_basis.get("trials") if isinstance(metric_basis, dict) else None
+ contributing = trials.get("with_skill") if isinstance(trials, dict) else None
+ excluded_attempts = exclusion.get("excluded_attempts") if complete else None
+ if type(contributing) is not int or type(excluded_attempts) is not int:
+ contributing = excluded_attempts = None
+ reason = "Cases whose arms were scored by different judge models are left out of this metric's lift"
+ return {
+ "excluded_cases": excluded_cases,
+ # Cutting can make two long keys equal; each is listed once.
+ "case_ids": sorted({case_id[:_MAX_LIFT_EXCLUDED_CASE_ID_CHARS] for case_id in case_ids})[
+ :_MAX_LIFT_EXCLUDED_CASE_IDS_PER_CARD
+ ],
+ "excluded_attempts": excluded_attempts,
+ "contributing_attempts": contributing,
+ "reason": f"{reason}; the arm scores include them." if arm_scores else f"{reason}.",
+ }
+
+
def _evaluator_cards(
evaluators: dict[str, dict[str, Any]],
*,
@@ -2953,6 +3093,8 @@ def _evaluator_cards(
custom_lift: dict[str, Any],
report_budget: _ReportBudget,
baseline_only: bool = False,
+ lift_exclusions: dict[str, dict[str, Any]] | None = None,
+ lift_basis: dict[str, Any] | None = None,
) -> list[dict[str, Any]]:
from skillevaluator.tier3.harbor.metrics import METRIC_DISPLAY
@@ -2977,6 +3119,8 @@ def _evaluator_cards(
rewards=rewards,
report_budget=report_budget,
baseline_only=baseline_only,
+ lift_exclusion=(lift_exclusions or {}).get(metric),
+ lift_basis=lift_basis,
)
)
diff --git a/src/skillevaluator/reporting/html.py b/src/skillevaluator/reporting/html.py
index 71d3a4e4..528b673a 100644
--- a/src/skillevaluator/reporting/html.py
+++ b/src/skillevaluator/reporting/html.py
@@ -822,6 +822,19 @@ def _sanitize_tier3_pass_at_k(value: object) -> dict[str, Any]:
return pass_at_k
+def _sanitize_tier3_comparison_basis(value: object) -> dict[str, Any] | None:
+ """Keep an evaluator card's judge-model lift exclusions only when they name a case count."""
+ if not isinstance(value, dict) or not (excluded_cases := _nonnegative_count(value.get("excluded_cases"))):
+ return None
+ return {
+ "excluded_cases": excluded_cases,
+ "case_ids": _tier3_identifier_list(value.get("case_ids")),
+ "excluded_attempts": _strict_nonnegative_count(value.get("excluded_attempts")),
+ "contributing_attempts": _strict_nonnegative_count(value.get("contributing_attempts")),
+ "reason": _json_safe_tier3_text(value["reason"]) if isinstance(value.get("reason"), str) else "",
+ }
+
+
def _sanitize_tier3_evaluator_card(
value: dict[str, Any],
*,
@@ -851,6 +864,8 @@ def _sanitize_tier3_evaluator_card(
for key in FAILURE_REASON_EVIDENCE_KEYS:
if not (isinstance(entry.get(key), str) and entry[key]):
entry.pop(key, None)
+ if entry.pop("comparison_excluded", None) is True:
+ entry["comparison_excluded"] = True
if (
include_evidence_unavailable_default
and card.get("id") in _METRIC_EVIDENCE_REF_METRICS
@@ -867,6 +882,9 @@ def _sanitize_tier3_evaluator_card(
entry.pop("evidence_unavailable_reason", None)
evidence.append(entry)
card["evidence"] = evidence
+ comparison_basis = _sanitize_tier3_comparison_basis(card.pop("comparison_basis", None))
+ if comparison_basis:
+ card["comparison_basis"] = comparison_basis
sampling = card.get("evidence_sampling") if isinstance(card.get("evidence_sampling"), dict) else {}
for key in ("represented_cases", "represented_trials", "total_trials", "scanned_trials"):
if key in sampling:
diff --git a/src/skillevaluator/reporting/templates/report.html.j2 b/src/skillevaluator/reporting/templates/report.html.j2
index 78175add..d42c9463 100644
--- a/src/skillevaluator/reporting/templates/report.html.j2
+++ b/src/skillevaluator/reporting/templates/report.html.j2
@@ -2375,6 +2375,8 @@
.t3-eval-evidence li .t3-ev-notes { color:var(--text-secondary); font-size:0.78rem; }
.t3-eval-evidence li .t3-ev-detail { grid-column:1 / -1; display:flex; flex-wrap:wrap; align-items:center; gap:0.5rem; }
.t3-ev-excerpt { font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,monospace; font-size:0.76rem; padding:0.15rem 0.4rem; border-radius:4px; background:var(--bg-tertiary); color:var(--text-secondary); overflow-wrap:anywhere; }
+ .t3-eval-evidence li .t3-ev-lift-excluded { color:var(--text-secondary); font-size:0.76rem; font-weight:600; }
+ .t3-eval-lift-note { max-width:16rem; }
.t3-score-reason { border-bottom:1px dotted currentColor; cursor:help; }
/* Attempt Details: SkillEvaluator-style chip table for per-case attempt scores */
@@ -3658,6 +3660,19 @@
{% set ev_lift_class = 'up' if card.lift >= tier3_lift_pass_threshold else ('down' if card.lift <= tier3_lift_fail_threshold else 'flat') %}
<div class="t3-delta {{ ev_lift_class }}" style="margin-top:0.4rem; display:inline-block;">{{ "%+.2f" | format(card.lift) }} lift</div>
{% endif %}
+ {% set lift_excluded_cases = card.comparison_basis.excluded_cases if card.comparison_basis else 0 %}
+ {% if lift_excluded_cases %}
+ {% set one_lift_excluded_case = lift_excluded_cases == 1 %}
+ {% set lift_excluded_label = lift_excluded_cases ~ (' case' if one_lift_excluded_case else ' cases') %}
+ {# Only a card with both arm scores can say they include the cases. #}
+ <div class="t3-dim-hint t3-eval-lift-note">
+ {%- if card.lift is none -%}
+ No lift: {{ lift_excluded_label }} {{ 'was' if one_lift_excluded_case else 'were' }} scored by different judge models and no comparable result remains
+ {%- else -%}
+ Lift leaves out {{ lift_excluded_label }} scored by different judge models
+ {%- endif -%}
+ {%- if card.baseline is not none %}; the arm scores include {{ 'it' if one_lift_excluded_case else 'them' }}{% endif %}.</div>
+ {% endif %}
</div>
</div>
<div class="t3-eval-bars">
@@ -3678,7 +3693,7 @@
{% set eid = ev.entry_id or 'trial' %}
{% if eid not in dedup_map %}
{% set _ = dedup_order.append(eid) %}
- {% set _ = dedup_map.update({eid: {'count': 0, 'sum': 0.0, 'with_score': 0, 'notes': [], 'failures': [], 'checks': [], 'evidence_refs': [], 'evidence_unavailable_reason': none, 'excerpt': none, 'reason': none, 'reason_score': none}}) %}
+ {% set _ = dedup_map.update({eid: {'count': 0, 'sum': 0.0, 'with_score': 0, 'notes': [], 'failures': [], 'checks': [], 'evidence_refs': [], 'evidence_unavailable_reason': none, 'excerpt': none, 'reason': none, 'reason_score': none, 'comparison_excluded': false}}) %}
{% endif %}
{% set entry = dedup_map[eid] %}
{% set ev_occurrences = ev.occurrences or 1 %}
@@ -3698,6 +3713,7 @@
{% endif %}
{% if ev.checks and not entry.checks %}{% set _ = entry.update({'checks': ev.checks}) %}{% endif %}
{% if ev.evidence_refs and not entry.evidence_refs %}{% set _ = entry.update({'evidence_refs': ev.evidence_refs}) %}{% endif %}
+ {% if ev.comparison_excluded %}{% set _ = entry.update({'comparison_excluded': true}) %}{% endif %}
{% if ev.evidence_unavailable_reason and not entry.evidence_unavailable_reason %}{% set _ = entry.update({'evidence_unavailable_reason': ev.evidence_unavailable_reason}) %}{% endif %}
{% endfor %}
{% set evidence_totals = namespace(trials=0) %}
@@ -3731,6 +3747,7 @@
{% endif %}
{% if ev.reason %}<div class="t3-ev-detail t3-ev-reason">{{ ev.reason }}</div>{% endif %}
{% if ev.excerpt %}<div class="t3-ev-detail"><code class="t3-ev-excerpt">{{ ev.excerpt }}</code></div>{% endif %}
+ {% if ev.comparison_excluded %}<div class="t3-ev-detail t3-ev-lift-excluded">Left out of lift: different judge models</div>{% endif %}
</li>
{% endfor %}
</ul>
diff --git a/tests/test_judge_lift_integrity.py b/tests/test_judge_lift_integrity.py
index f58ddad2..fec57b1d 100644
--- a/tests/test_judge_lift_integrity.py
+++ b/tests/test_judge_lift_integrity.py
@@ -6,13 +6,27 @@
from __future__ import annotations
import json
+import re
from pathlib import Path
import pytest
-from skillevaluator.evaluation.tier3_report import agent_eval_result_from_directory
+from skillevaluator.evaluation import tier3_report
+from skillevaluator.evaluation.tier3_report import (
+ _validation_result_from_payload,
+ agent_eval_result_from_directory,
+ build_agent_eval_payload,
+ render_agent_eval_html_report,
+)
+from skillevaluator.reporting import HTMLReporter
+from skillevaluator.reporting.html import _sanitize_tier3_display_payload
from skillevaluator.tier3.harbor.collector import _extract_rewards, collect_harbor_results
-from skillevaluator.tier3.harbor.metrics import DEFAULT_METRIC_SET, DEFAULT_METRICS, DEFAULT_SCORE_POLICY
+from skillevaluator.tier3.harbor.metrics import (
+ DEFAULT_METRIC_SET,
+ DEFAULT_METRICS,
+ DEFAULT_SCORE_POLICY,
+ mark_not_applicable,
+)
def _write_json(path: Path, payload: dict) -> None:
@@ -322,3 +336,456 @@ def test_native_aggregate_keeps_numeric_authority_and_only_declared_step_judges(
assert rows[0]["accuracy"] == 0.25
assert rows[0]["_judge_models"] == {"accuracy": ["fallback", "primary"]}
assert rows[0]["details"]["accuracy"]["judge_models"] == ["fallback", "primary"]
+
+
+# Matches the note in every wording; the stylesheet names the class without this attribute.
+_LIFT_NOTE_DIV = 'class="t3-dim-hint t3-eval-lift-note"'
+_TRIAL_LABEL = "Left out of lift: different judge models"
+_BASIS_REASON = (
+ "Cases whose arms were scored by different judge models are left out of this metric's lift; "
+ "the arm scores include them."
+)
+
+
+def _card(agent: dict, metric: str = "accuracy") -> dict:
+ return next(card for card in agent["evaluator_cards"] if card["id"] == metric)
+
+
+def _card_html(html: str, label: str) -> str:
+ """Return one evaluator card's markup, failing unless its ``div`` elements close."""
+ for match in re.finditer(r'<div class="t3-eval-card ', html):
+ depth = 0
+ for tag in re.finditer(r"<(/?)div\b", html[match.start() :]):
+ depth += -1 if tag.group(1) else 1
+ if depth == 0:
+ card = html[match.start() : match.start() + tag.end()]
+ break
+ else:
+ raise AssertionError("unbalanced evaluator card markup")
+ if f'<div class="t3-eval-title">{label}</div>' in card:
+ return card
+ raise AssertionError(f"no {label} evaluator card")
+
+
+def _text(fragment: str) -> str:
+ return re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", fragment)).strip()
+
+
+def _render(payload: dict) -> str:
+ result = _validation_result_from_payload(payload)
+ assert result is not None
+ return HTMLReporter(include_timestamp=False).render_all([result])
+
+
[email protected]("all_mixed", [False, True])
+def test_evaluator_card_names_the_cases_its_lift_left_out(tmp_path: Path, all_mixed: bool) -> None:
+ _, run = _collect(tmp_path, all_mixed=all_mixed)
+ report = _report(run)
+ agent = report["agents"]["codex"]
+ mixed = ["case-1", "case-2"] if all_mixed else ["case-1"]
+ evaluator = agent["evaluators"]["accuracy"]
+ # The arm scores keep every scored case; only the lift leaves the mixed cases out.
+ assert (evaluator["with_skill"], evaluator["baseline"]) == (0.5, 0.2)
+ assert evaluator["lift"] == (None if all_mixed else -0.4)
+ assert evaluator["lift_excluded_cases"] == len(mixed)
+ assert all(
+ "lift_excluded_cases" not in agent["evaluators"][metric] for metric in DEFAULT_METRICS if metric != "accuracy"
+ )
+ card = _card(agent)
+ assert (card["with_skill"], card["baseline"], card["lift"]) == (0.5, 0.2, evaluator["lift"])
+ assert card["comparison_basis"] == {
+ "excluded_cases": len(mixed),
+ "case_ids": mixed,
+ "excluded_attempts": len(mixed),
+ "contributing_attempts": 0 if all_mixed else 1,
+ "reason": _BASIS_REASON,
+ }
+ assert [item["entry_id"] for item in card["evidence"] if item.get("comparison_excluded")] == mixed
+ assert all("comparison_basis" not in other for other in agent["evaluator_cards"] if other["id"] != "accuracy")
+ assert _card(report) == card
+
+ skill = tmp_path / "demo"
+ skill.mkdir()
+ html = render_agent_eval_html_report(skill, run, use_llm_judge=False).read_text(encoding="utf-8")
+ card_html = _card_html(html, "Accuracy")
+ note = (
+ "No lift: 2 cases were scored by different judge models and no comparable result remains; "
+ "the arm scores include them."
+ if all_mixed
+ else "Lift leaves out 1 case scored by different judge models; the arm scores include it."
+ )
+ assert note in _text(card_html)
+ assert ("Lift leaves out" in _text(card_html)) is not all_mixed
+ assert ("-0.40 lift" in _text(card_html)) is not all_mixed
+ assert card_html.count(_TRIAL_LABEL) == len(mixed)
+ assert html.count(_LIFT_NOTE_DIV) == 1
+
+
+def _attempt(case: str, trial_id: str, accuracy: float | None, judge: str | None) -> dict:
+ reward = {**dict.fromkeys(DEFAULT_METRICS, 0.5), "entry_id": case, "trial_id": trial_id, "details": {}}
+ if accuracy is None:
+ mark_not_applicable(reward, "accuracy")
+ else:
+ reward.update(accuracy=accuracy, details={"accuracy": {"score": accuracy, "judge_model_used": judge}})
+ return reward
+
+
+def _info(with_rewards: list[dict], baseline_rewards: list[dict]) -> dict:
+ succeeded = {"execution_status": "succeeded"}
+ return {
+ "execution_status": "succeeded",
+ "expected_attempts": len(with_rewards) + len(baseline_rewards),
+ "scored_attempts": len(with_rewards) + len(baseline_rewards),
+ "num_trials": len(with_rewards),
+ "num_trials_baseline": len(baseline_rewards),
+ "with_skill": {**dict.fromkeys(DEFAULT_METRICS, 0.5), "accuracy": 0.5},
+ "without_skill": dict.fromkeys(DEFAULT_METRICS, 0.5),
+ "rewards": with_rewards,
+ "rewards_baseline": baseline_rewards,
+ "rewards_complete": True,
+ "rewards_baseline_complete": True,
+ "conditions": {"with_skill": succeeded, "without_skill": succeeded},
+ }
+
+
+def test_comparison_basis_counts_repeated_attempts_apart_from_na_and_unpaired_cases() -> None:
+ info = _info(
+ [
+ _attempt("kept", "kept__a1", 1.0, "primary"),
+ _attempt("mixed", "mixed__a1", 0.0, "fallback"),
+ _attempt("mixed", "mixed__a2", 0.5, "primary"),
+ _attempt("mixed", "mixed__a3", None, None),
+ ],
+ [
+ _attempt("kept", "kept__b1", 0.5, "primary"),
+ _attempt("mixed", "mixed__b1", 0.5, "primary"),
+ # A case only the baseline scored is unpaired, not judge-mixed.
+ _attempt("baseline-only", "baseline-only__b1", 0.5, "fallback"),
+ ],
+ )
+ payload = build_agent_eval_payload("demo", {"codex": info}, use_llm_judge=False)
+ assert payload is not None
+ agent = payload["agents"]["codex"]
+ assert agent["evaluators"]["accuracy"]["lift"] == 0.5
+ assert agent["evaluators"]["accuracy"]["lift_excluded_cases"] == 1
+ basis = _card(agent)["comparison_basis"]
+ # Two scored attempts of the mixed case are left out; its N/A attempt never had a score.
+ assert (basis["case_ids"], basis["excluded_attempts"], basis["contributing_attempts"]) == (["mixed"], 2, 1)
+ flagged = {item["entry_id"]: item.get("comparison_excluded", False) for item in _card(agent)["evidence"]}
+ assert flagged == {"kept": False, "mixed": True}
+
+
+def test_matching_judges_add_no_lift_note() -> None:
+ info = _info(
+ [_attempt("case-1", "case-1__a1", 1.0, "primary"), _attempt("case-2", "case-2__a1", 0.0, "primary")],
+ [_attempt("case-1", "case-1__b1", 0.5, "primary"), _attempt("case-2", "case-2__b1", 0.5, "primary")],
+ )
+ payload = build_agent_eval_payload("demo", {"codex": info}, use_llm_judge=False)
+ assert payload is not None
+ agent = payload["agents"]["codex"]
+ assert agent["evaluators"]["accuracy"]["lift"] == 0.0
+ assert all("lift_excluded_cases" not in evaluator for evaluator in agent["evaluators"].values())
+ assert all("comparison_basis" not in card for card in agent["evaluator_cards"])
+ assert all("comparison_excluded" not in item for card in agent["evaluator_cards"] for item in card["evidence"])
+ html = _render(payload)
+ assert _LIFT_NOTE_DIV not in html
+ assert _TRIAL_LABEL not in html
+
+
+def test_saved_run_without_the_mixed_case_rewards_shows_only_the_recorded_count(tmp_path: Path) -> None:
+ collected, run = _collect(tmp_path)
+ lift_path = run / "codex" / "lift.json"
+ legacy_lift = json.loads(lift_path.read_text(encoding="utf-8"))
+ legacy_lift.pop("basis", None)
+ _write_json(lift_path, legacy_lift)
+ for reward in (run / "codex" / "with-skill" / "trials").glob("*/reward.json"):
+ if json.loads(reward.read_text(encoding="utf-8"))["entry_id"] == "case-1":
+ reward.unlink()
+ agent = _report(run)["agents"]["codex"]
+ # The recorded lift and its exclusion count stand; no loaded reward names the case.
+ assert agent["evaluators"]["accuracy"]["lift"] == collected["agents"]["codex"]["lift"]["accuracy"]["delta"]
+ assert agent["evaluators"]["accuracy"]["lift_excluded_cases"] == 1
+ card = _card(agent)
+ assert card["comparison_basis"] == {
+ "excluded_cases": 1,
+ "case_ids": [],
+ "excluded_attempts": None,
+ "contributing_attempts": None,
+ "reason": _BASIS_REASON,
+ }
+ assert [item["entry_id"] for item in card["evidence"]] == ["case-2"]
+ assert not any(item.get("comparison_excluded") for item in card["evidence"])
+
+ skill = tmp_path / "demo"
+ skill.mkdir()
+ card_html = _card_html(
+ render_agent_eval_html_report(skill, run, use_llm_judge=False).read_text(encoding="utf-8"), "Accuracy"
+ )
+ assert "Lift leaves out 1 case scored by different judge models" in _text(card_html)
+ assert _TRIAL_LABEL not in card_html
+
+
+def test_trimmed_card_evidence_keeps_the_lift_note(monkeypatch: pytest.MonkeyPatch) -> None:
+ info = _info(
+ [_attempt("case-1", "case-1__a1", 1.0, "fallback"), _attempt("case-2", "case-2__a1", 0.0, "primary")],
+ [_attempt("case-1", "case-1__b1", 0.5, "primary"), _attempt("case-2", "case-2__b1", 0.5, "primary")],
+ )
+ real_size = tier3_report._serialized_payload_size
+
+ def size_until_evidence_trimmed(payload: dict) -> int:
+ cards = [card for agent in payload.get("agents", {}).values() for card in agent.get("evaluator_cards", [])]
+ evidence = any(card.get("evidence") for card in cards)
+ return real_size(payload) + (tier3_report._MAX_EMBEDDED_REPORT_BYTES if evidence else 0)
+
+ monkeypatch.setattr(tier3_report, "_serialized_payload_size", size_until_evidence_trimmed)
+ payload = build_agent_eval_payload("demo", {"codex": info}, use_llm_judge=False)
+ assert payload is not None
+ assert payload["report_truncation"]["omitted"]["evidence_entries"] > 0
+ card = _card(payload["agents"]["codex"])
+ assert card["evidence"] == []
+ assert card["comparison_basis"]["excluded_cases"] == 1
+ card_html = _card_html(_render(payload), "Accuracy")
+ assert "Lift leaves out 1 case scored by different judge models" in _text(card_html)
+ assert _TRIAL_LABEL not in card_html
+
+
+def _mixed_info() -> dict:
+ """One case scored by a fallback judge in the with-skill arm, one by matching judges."""
+ return _info(
+ [_attempt("case-1", "case-1__a1", 1.0, "fallback"), _attempt("case-2", "case-2__a1", 0.0, "primary")],
+ [_attempt("case-1", "case-1__b1", 0.5, "primary"), _attempt("case-2", "case-2__b1", 0.5, "primary")],
+ )
+
+
+def test_never_activated_agent_gets_no_lift_note() -> None:
+ info = {**_mixed_info(), "skill_activation": {"status": "never_activated"}}
+ payload = build_agent_eval_payload("demo", {"codex": info}, use_llm_judge=False)
+ assert payload is not None
+ agent = payload["agents"]["codex"]
+ assert agent["verdict"] == tier3_report.VERDICT_INCONCLUSIVE
+ # The agent has no lift at all, so nothing explains what its lift left out.
+ assert agent["evaluators"]["accuracy"]["lift"] is None
+ assert "lift_excluded_cases" not in agent["evaluators"]["accuracy"]
+ card = _card(agent)
+ assert card["lift"] is None
+ assert "comparison_basis" not in card
+ assert not any(item.get("comparison_excluded") for item in card["evidence"])
+ html = _render(payload)
+ assert _LIFT_NOTE_DIV not in html
+ assert _TRIAL_LABEL not in html
+
+
+ "basis",
+ [
+ {"excluded_cases": True},
+ {"excluded_cases": "3"},
+ {"excluded_cases": -1},
+ {"excluded_cases": 2**70},
+ {"excluded_cases": 1.0},
+ ["excluded_cases", 1],
+ ],
+)
+def test_html_ignores_a_malformed_comparison_basis_and_flag(basis: object) -> None:
+ info = _info(
+ [_attempt("case-1", "case-1__a1", 1.0, "primary"), _attempt("case-2", "case-2__a1", 0.0, "primary")],
+ [_attempt("case-1", "case-1__b1", 0.5, "primary"), _attempt("case-2", "case-2__b1", 0.5, "primary")],
+ )
+ payload = build_agent_eval_payload("demo", {"codex": info}, use_llm_judge=False)
+ assert payload is not None
+ for card in (_card(payload["agents"]["codex"]), _card(payload)):
+ card["comparison_basis"] = basis
+ for item in card["evidence"]:
+ item["comparison_excluded"] = "yes"
+ html = _render(payload)
+ assert _LIFT_NOTE_DIV not in html
+ assert _TRIAL_LABEL not in html
+
+
+def test_html_keeps_comparison_basis_fields_only_in_their_types() -> None:
+ payload = build_agent_eval_payload("demo", {"codex": _mixed_info()}, use_llm_judge=False)
+ assert payload is not None
+ for attempts in (True, "1", -1, 2**70, 1.0):
+ card = _card(payload["agents"]["codex"])
+ card["comparison_basis"] = {
+ "excluded_cases": 1,
+ "case_ids": "case-1",
+ "excluded_attempts": attempts,
+ "contributing_attempts": attempts,
+ "reason": 5,
+ }
+ card["evidence"][0]["comparison_excluded"] = 1
+ sanitized = _card(_sanitize_tier3_display_payload(payload)["agents"]["codex"])
+ assert sanitized["comparison_basis"] == {
+ "excluded_cases": 1,
+ "case_ids": [],
+ "excluded_attempts": None,
+ "contributing_attempts": None,
+ "reason": "",
+ }
+ assert not any("comparison_excluded" in item for item in sanitized["evidence"])
+
+
+def test_comparison_basis_caps_case_ids_and_lists_each_cut_id_once() -> None:
+ with_rewards = [_attempt("kept", "kept__a1", 1.0, "primary")]
+ baseline_rewards = [_attempt("kept", "kept__b1", 0.5, "primary")]
+ for index in range(70):
+ case = f"case-{index:03d}-" + "x" * 300
+ with_rewards.append(_attempt(case, f"{case}__a1", 1.0, "fallback"))
+ baseline_rewards.append(_attempt(case, f"{case}__b1", 0.5, "primary"))
+ # Two keys that match in their first 256 characters and sort first.
+ shared_prefix = "a" * 260
+ for case in (f"{shared_prefix}-a", f"{shared_prefix}-b"):
+ with_rewards.append(_attempt(case, f"{case}__a1", 1.0, "fallback"))
+ baseline_rewards.append(_attempt(case, f"{case}__b1", 0.5, "primary"))
+ payload = build_agent_eval_payload("demo", {"codex": _info(with_rewards, baseline_rewards)}, use_llm_judge=False)
+ assert payload is not None
+ agent = payload["agents"]["codex"]
+ assert agent["evaluators"]["accuracy"]["lift_excluded_cases"] == 72
+ basis = _card(agent)["comparison_basis"]
+ # The count stays exact while the list stops at 64 cut keys, and the two
+ # keys that match once cut appear once.
+ assert basis["excluded_cases"] == 72
+ assert basis["case_ids"] == ["a" * 256, *(f"case-{index:03d}-".ljust(256, "x") for index in range(63))]
+ assert len(set(basis["case_ids"])) == 64
+ assert (basis["excluded_attempts"], basis["contributing_attempts"]) == (72, 1)
+
+
+def test_comparison_basis_leaves_out_trials_without_a_resolved_case() -> None:
+ info = _info(
+ [
+ _attempt("kept", "kept__a1", 1.0, "primary"),
+ _attempt("mixed", "mixed__a1", 0.0, "fallback"),
+ {**_attempt("", "unresolved__a1", 1.0, "fallback"), "entry_id": ""},
+ ],
+ [
+ _attempt("kept", "kept__b1", 0.5, "primary"),
+ _attempt("mixed", "mixed__b1", 0.5, "primary"),
+ {**_attempt("", "unresolved__b1", 0.5, "primary"), "entry_id": ""},
+ ],
+ )
+ payload = build_agent_eval_payload("demo", {"codex": info}, use_llm_judge=False)
+ assert payload is not None
+ agent = payload["agents"]["codex"]
+ # Lift pairing never compares a trial without a case, so it is not a left-out case.
+ assert agent["evaluators"]["accuracy"]["lift_excluded_cases"] == 1
+ card = _card(agent)
+ basis = card["comparison_basis"]
+ assert (basis["case_ids"], basis["excluded_attempts"], basis["contributing_attempts"]) == (["mixed"], 1, 1)
+ assert [item["entry_id"] for item in card["evidence"] if item.get("comparison_excluded")] == ["mixed"]
+
+
+def _split_trial_info() -> dict:
+ """A with-skill trial whose rows name two cases: the logical load is incomplete."""
+ return _info(
+ [
+ _attempt("mixed", "mixed__a1", 1.0, "fallback"),
+ _attempt("kept", "kept__a1", 0.0, "primary"),
+ _attempt("odd-a", "shared", 0.5, "primary"),
+ _attempt("odd-b", "shared", 0.5, "primary"),
+ ],
+ [
+ _attempt("mixed", "mixed__b1", 0.5, "primary"),
+ _attempt("kept", "kept__b1", 0.5, "primary"),
+ _attempt("odd-a", "odd-a__b1", 0.5, "primary"),
+ _attempt("odd-b", "odd-b__b1", 0.5, "primary"),
+ ],
+ )
+
+
+def _recorded_count_info() -> dict:
+ """``lift.json`` records two mixed cases; the loaded rewards identify one."""
+ return {**_mixed_info(), "lift": {"basis": {}, "judge_mixed_cases": {"accuracy": 2}}}
+
+
+ ("make_info", "excluded_cases"),
+ [(_split_trial_info, 1), (_recorded_count_info, 2)],
+ ids=["trial-rows-name-two-cases", "recorded-count-exceeds-identified-cases"],
+)
+def test_attempt_counts_stay_null_unless_the_load_identifies_every_left_out_case(
+ make_info, excluded_cases: int
+) -> None:
+ payload = build_agent_eval_payload("demo", {"codex": make_info()}, use_llm_judge=False)
+ assert payload is not None
+ agent = payload["agents"]["codex"]
+ # Both arms' rewards loaded and the lift basis is published, but it does not
+ # describe every left-out case, so neither attempt count is published.
+ assert agent["lift_basis"]["metrics"]["accuracy"]["trials"]["with_skill"] == 1
+ assert agent["evaluators"]["accuracy"]["lift"] is None
+ assert agent["evaluators"]["accuracy"]["lift_excluded_cases"] == excluded_cases
+ basis = _card(agent)["comparison_basis"]
+ assert basis["excluded_cases"] == excluded_cases
+ assert basis["case_ids"] == ["case-1" if excluded_cases == 2 else "mixed"]
+ assert (basis["excluded_attempts"], basis["contributing_attempts"]) == (None, None)
+ card_html = _card_html(_render(payload), "Accuracy")
+ noun = "1 case was" if excluded_cases == 1 else "2 cases were"
+ assert (
+ f"No lift: {noun} scored by different judge models and no comparable result remains; "
+ f"the arm scores include {'it' if excluded_cases == 1 else 'them'}."
+ ) in _text(card_html)
+
+
+def test_lift_note_mentions_arm_scores_only_when_both_arms_have_one() -> None:
+ info = _mixed_info()
+ # The baseline arm failed, so it has no arm score, although its rewards loaded.
+ info["conditions"]["without_skill"] = {"execution_status": "failed"}
+ payload = build_agent_eval_payload("demo", {"codex": info}, use_llm_judge=False)
+ assert payload is not None
+ agent = payload["agents"]["codex"]
+ evaluator = agent["evaluators"]["accuracy"]
+ assert (evaluator["baseline"], evaluator["lift_excluded_cases"]) == (None, 1)
+ basis = _card(agent)["comparison_basis"]
+ assert (
+ basis["reason"] == "Cases whose arms were scored by different judge models are left out of this metric's lift."
+ )
+ card_html = _text(_card_html(_render(payload), "Accuracy"))
+ assert "Lift leaves out 1 case scored by different judge models." in card_html
+ assert "the arm scores include" not in card_html
+
+
+ ("recorded", "published"),
+ [(3, 3), (2**53 - 1, 2**53 - 1), (2**53, None), (10**30, None)],
+)
+def test_recorded_exclusion_count_is_published_only_when_json_carries_it_exactly(
+ recorded: int, published: int | None
+) -> None:
+ info = _info(
+ [_attempt("case-1", "case-1__a1", 1.0, "primary"), _attempt("case-2", "case-2__a1", 0.0, "primary")],
+ [_attempt("case-1", "case-1__b1", 0.5, "primary"), _attempt("case-2", "case-2__b1", 0.5, "primary")],
+ )
+ info.update(lift={"judge_mixed_cases": {"accuracy": recorded}}, rewards_complete=False)
+ payload = build_agent_eval_payload("demo", {"codex": info}, use_llm_judge=False)
+ assert payload is not None
+ agent = payload["agents"]["codex"]
+ assert agent["evaluators"]["accuracy"].get("lift_excluded_cases", None) == published
+ assert ("lift_excluded_cases" in agent["evaluators"]["accuracy"]) is (published is not None)
+ card = _card(agent)
+ assert ("comparison_basis" in card) is (published is not None)
+ if published is not None:
+ assert card["comparison_basis"]["excluded_cases"] == published
+
+
+ ("complete", "metrics_basis", "attempts"),
+ [
+ (True, {"accuracy": {"cases": 1, "trials": {"with_skill": 1, "without_skill": 1}}}, (1, 1)),
+ (False, {"accuracy": {"cases": 1, "trials": {"with_skill": 1, "without_skill": 1}}}, (None, None)),
+ # A metric the basis does not describe publishes neither count.
+ (True, {}, (None, None)),
+ (True, {"accuracy": {"cases": 1, "trials": {"with_skill": "1"}}}, (None, None)),
+ ],
+)
+def test_comparison_basis_publishes_both_attempt_counts_or_neither(
+ complete: bool, metrics_basis: dict, attempts: tuple[int | None, int | None]
+) -> None:
+ basis = tier3_report._card_comparison_basis(
+ "accuracy",
+ 1,
+ {"case-1"},
+ {"case_ids": {"case-1"}, "excluded_attempts": 1, "complete": complete},
+ {"metrics": metrics_basis},
+ arm_scores=True,
+ )
+ assert (basis["excluded_attempts"], basis["contributing_attempts"]) == attempts