Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

feat(tier3): report pass@k uncertainty and paired outcomes (#63)

Adem Vessell009aa300be79
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 74aad1f1..a2ad6515 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,12 +4,23 @@ All notable changes to SkillEvaluator are documented in this file.
## Unreleased
+### Fixed
+
+- Tier 3 paired pass@k evidence now respects Python's active integer-string
+ conversion limit, preserves nonzero Wilson interval widths and paired-effect
+ directions at large case counts, and documents exact-rational omission
+ markers.
+
## 0.2.1 - 2026-08-24
### Added
- Added a public benchmark publication gate, regression coverage, and a
documented rollout plan for generated `BENCHMARK.md` cards.
+- Tier 3 pass@k results now include per-arm 95% Wilson score intervals and,
+ when case identities pair completely, direction-preserving paired outcomes
+ with a two-sided exact McNemar diagnostic, its attainable-p resolution limit,
+ and the paired pass-rate delta.
### Changed
diff --git a/docs/reports.mdx b/docs/reports.mdx
index 45a05bd2..e6466eb6 100644
--- a/docs/reports.mdx
+++ b/docs/reports.mdx
@@ -237,7 +237,30 @@ scores. With `--n-attempts k`, each eval case runs k times per arm, and a case
counts as passed when at least one attempt clears the `--pass-threshold`
score. Comparing pass@k across arms (in `pass_at_k_lift.json`) tells you
whether the skill makes success more *repeatable*, not just whether the average
-score moved.
+score moved. Each arm also records a case-level 95% Wilson score interval for
+its pass rate. When both arms contain the same identified cases,
+`pass_at_k_lift.json` records the paired outcome counts — skill-only passes,
+baseline-only passes, both-pass, and neither-pass — plus a two-sided exact
+McNemar diagnostic over the discordant pairs. Partial or unidentified pairing
+is labeled and does not receive that exact-test result.
+
+The paired record also reports its pass-rate delta and the minimum p-value the
+exact test could attain with the observed number of discordant pairs. When that
+minimum is above 0.05, the report labels the test resolution-limited at that
+threshold. This is a mathematical resolution limit, not evidence of no effect;
+with five or fewer discordant pairs, a two-sided exact result cannot fall below
+0.05 regardless of direction. The JSON always preserves compact decimal text
+for both probabilities. It also records a reduced rational in each `*_exact`
+field when the numerator and denominator fit both SkillEvaluator's 4,300-digit
+safety cap and Python's active integer-string conversion limit. Otherwise that
+field is `null`, `*_omitted` is `true`, and `*_omitted_reason` is
+`"decimal_digit_limit"`. The numeric approximation is `null`, rather than a
+false zero, if binary floating-point conversion underflows.
+
+These uncertainty fields describe the cases in one configured run. They do not
+make heterogeneous cases independent, correct dataset leakage, or promote a
+small p-value into proof that a skill generalizes. Keep the dataset, attempt
+policy, agent, model, and environment fixed when interpreting them.
## Browse results
diff --git a/src/skillevaluator/evaluation/tier3_report.py b/src/skillevaluator/evaluation/tier3_report.py
index 27dee2ef..0dc50134 100644
--- a/src/skillevaluator/evaluation/tier3_report.py
+++ b/src/skillevaluator/evaluation/tier3_report.py
@@ -65,6 +65,7 @@ _MAX_RAW_TRIAL_REWARDS_TOTAL = 256
_MAX_RAW_METRICS_PER_REWARD = 64
_MAX_RAW_REWARD_FIELDS = 96
_MAX_CUSTOM_METRIC_NAME_VISITS_PER_REWARD = 128
+_MAX_UNPAIRED_CASE_IDS_IN_REPORT = 64
_MAX_EMBEDDED_REPORT_BYTES = 2 * 1024 * 1024
@@ -123,6 +124,7 @@ class _ReportBudget:
"raw_trial_rewards": _MAX_RAW_TRIAL_REWARDS_TOTAL,
"raw_metrics_per_reward": _MAX_RAW_METRICS_PER_REWARD,
"custom_metric_name_visits_per_reward": _MAX_CUSTOM_METRIC_NAME_VISITS_PER_REWARD,
+ "unpaired_case_id_samples": _MAX_UNPAIRED_CASE_IDS_IN_REPORT,
},
"omitted": dict(sorted(self.omitted.items())),
}
@@ -923,6 +925,46 @@ def _prune_non_best_agent_details(payload: dict[str, Any], report_budget: _Repor
report_budget.omit("non_best_agent_details", omitted)
+def _prune_pass_at_k_pairing_diagnostics(payload: dict[str, Any], report_budget: _ReportBudget) -> None:
+ """Bound legacy full mismatch-ID arrays while preserving counts and samples."""
+ pass_at_k_payloads = [payload.get("pass_at_k")]
+ agents = payload.get("agents")
+ if isinstance(agents, dict):
+ pass_at_k_payloads.extend(agent.get("pass_at_k") for agent in agents.values() if isinstance(agent, dict))
+
+ seen: set[int] = set()
+ omitted = 0
+ for pass_at_k in pass_at_k_payloads:
+ if not isinstance(pass_at_k, dict):
+ continue
+ lift = pass_at_k.get("lift")
+ paired = lift.get("paired_comparison") if isinstance(lift, dict) else None
+ if not isinstance(paired, dict) or id(paired) in seen:
+ continue
+ seen.add(id(paired))
+
+ for condition in ("with_skill", "without_skill"):
+ ids_key = f"{condition}_unpaired_case_ids"
+ count_key = f"{condition}_unpaired_case_count"
+ truncated_key = f"{ids_key}_truncated"
+ case_ids = paired.get(ids_key)
+ if not isinstance(case_ids, list):
+ continue
+
+ declared_count = paired.get(count_key)
+ if not isinstance(declared_count, int) or isinstance(declared_count, bool) or declared_count < 0:
+ declared_count = 0
+ paired[count_key] = max(declared_count, len(case_ids))
+ if len(case_ids) > _MAX_UNPAIRED_CASE_IDS_IN_REPORT:
+ omitted += len(case_ids) - _MAX_UNPAIRED_CASE_IDS_IN_REPORT
+ paired[ids_key] = case_ids[:_MAX_UNPAIRED_CASE_IDS_IN_REPORT]
+ paired[truncated_key] = True
+ else:
+ paired[truncated_key] = bool(paired.get(truncated_key))
+
+ report_budget.omit("unpaired_case_ids", omitted)
+
+
def _enforce_report_payload_budget(payload: dict[str, Any], report_budget: _ReportBudget) -> None:
"""Keep the complete self-contained payload within a hard serialized budget.
@@ -936,6 +978,11 @@ def _enforce_report_payload_budget(payload: dict[str, Any], report_budget: _Repo
if report_budget.truncated:
payload["report_truncation"] = report_budget.signal()
+ # Older artifacts may contain every unmatched case identifier. Normalize
+ # them before the size check so a report cannot discard all agents and
+ # pass@k truth merely because diagnostic IDs were duplicated into the
+ # best-agent and top-level projections.
+ _prune_pass_at_k_pairing_diagnostics(payload, report_budget)
refresh_signal()
if _serialized_payload_size(payload) <= _MAX_EMBEDDED_REPORT_BYTES:
return
diff --git a/src/skillevaluator/reporting/html.py b/src/skillevaluator/reporting/html.py
index 3ecf802c..8bd15a5c 100644
--- a/src/skillevaluator/reporting/html.py
+++ b/src/skillevaluator/reporting/html.py
@@ -22,6 +22,7 @@ from __future__ import annotations
import base64
import hashlib
import json
+import math
import pkgutil
from dataclasses import dataclass
from datetime import UTC, datetime
@@ -163,6 +164,76 @@ def _related_paths(finding: object) -> list[str]:
return paths
+def _adaptive_unsigned_percent_decimals(percentage: float) -> int:
+ """Return the existing bounded precision policy for one unsigned percentage."""
+ if percentage in {0.0, 100.0}:
+ return 0
+ if 0 < percentage < 1 or 99 < percentage < 100:
+ distance_from_endpoint = min(percentage, 100 - percentage)
+ return max(2, min(6, math.ceil(-math.log10(distance_from_endpoint))))
+ return 0
+
+
+def _adaptive_percent(value: object, signed: bool = False) -> str:
+ """Format percentages without rounding narrow nonzero effects to zero or endpoints."""
+ try:
+ percentage = float(value) * 100
+ except (TypeError, ValueError, OverflowError):
+ return "—"
+ if not math.isfinite(percentage):
+ return "—"
+
+ if signed:
+ magnitude = abs(percentage)
+ if magnitude == 0 or magnitude >= 0.1:
+ decimals = 1
+ else:
+ decimals = max(2, min(6, math.ceil(-math.log10(magnitude))))
+ return f"{percentage:+.{decimals}f}%"
+
+ decimals = _adaptive_unsigned_percent_decimals(percentage)
+ return f"{percentage:.{decimals}f}%"
+
+
+def _adaptive_interval_percent(interval: object) -> str:
+ """Format a percentage interval with shared precision and visible bounds."""
+ if not isinstance(interval, dict):
+ return "—"
+ try:
+ lower = float(interval["lower"]) * 100
+ upper = float(interval["upper"]) * 100
+ except (KeyError, TypeError, ValueError, OverflowError):
+ return "—"
+ if not math.isfinite(lower) or not math.isfinite(upper) or lower > upper:
+ return "—"
+
+ decimals = max(
+ _adaptive_unsigned_percent_decimals(lower),
+ _adaptive_unsigned_percent_decimals(upper),
+ )
+ if lower != upper:
+ for candidate in range(decimals, 7):
+ if f"{lower:.{candidate}f}" != f"{upper:.{candidate}f}":
+ decimals = candidate
+ break
+ else:
+ # Preserve a distinct interval narrower than the six-decimal
+ # display budget by rounding its bounds outward at that budget.
+ decimals = 6
+ scale = 10**decimals
+ lower = math.floor(lower * scale) / scale
+ upper = math.ceil(upper * scale) / scale
+
+ def _bound(value: float) -> str:
+ # Exact mathematical endpoints remain compact even when the other
+ # bound needs additional precision.
+ if value in {0.0, 100.0}:
+ return f"{value:.0f}%"
+ return f"{value:.{decimals}f}%"
+
+ return f"{_bound(lower)}\N{EN DASH}{_bound(upper)}"
+
+
class PackageLoader(BaseLoader):
"""Custom Jinja2 loader that loads templates from package resources."""
@@ -223,6 +294,8 @@ class HTMLReporter(ReporterBase):
loader = PackageLoader("skillevaluator.reporting", "templates")
environment = Environment(loader=loader, autoescape=True)
environment.filters["related_paths"] = _related_paths
+ environment.filters["adaptive_percent"] = _adaptive_percent
+ environment.filters["adaptive_interval_percent"] = _adaptive_interval_percent
return environment
@staticmethod
diff --git a/src/skillevaluator/reporting/templates/report.html.j2 b/src/skillevaluator/reporting/templates/report.html.j2
index 010088a2..6b344c14 100644
--- a/src/skillevaluator/reporting/templates/report.html.j2
+++ b/src/skillevaluator/reporting/templates/report.html.j2
@@ -1212,8 +1212,9 @@
box-shadow: var(--shadow);
}
.chart-card h3 { font-size: 0.95rem; color: var(--text-secondary); margin-bottom: 0.75rem; }
- .table-scroll { overflow-x: auto; margin: 1rem 0; border-radius: 10px; border: 1px solid var(--border-color); }
+ .table-scroll { max-width: 100%; overflow-x: auto; margin: 1rem 0; border-radius: 10px; border: 1px solid var(--border-color); -webkit-overflow-scrolling: touch; }
.table-scroll .summary-table { margin: 0; }
+ .pass-at-k-table { min-width: 720px; }
.metric-bar-cell { display: flex; align-items: center; gap: 0.6rem; min-width: 120px; }
.metric-value { min-width: 2.5rem; font-weight: 700; }
.mini-bar { width: 76px; height: 7px; background: var(--bg-tertiary); border-radius: 999px; overflow: hidden; }
@@ -2577,11 +2578,12 @@
{{ t3_max_attempts }} attempt{{ 's' if t3_max_attempts != 1 else '' }}, where a case counts as
"passed" only if the trial score reaches the configured pass threshold (currently
{{ "%.2f" | format(t3_attempt_policy.pass_threshold or 0.5) }}).
- Higher rates and a positive lift mean the skill consistently helps the agent reach the goal.
+ Higher rates and a positive lift show better observed repeatability on these evaluated cases.
</p>
- <table class="summary-table" style="width:100%;">
+ <div class="table-scroll" tabindex="0" role="region" aria-label="Pass at K summary table">
+ <table class="summary-table pass-at-k-table" style="width:100%;">
<thead><tr>
- <th>Agent</th><th>Condition</th><th>Pass Rate</th><th>Cases Passed</th><th>Scored Attempts</th><th>Lift</th>
+ <th>Agent</th><th>Condition</th><th>Pass Rate</th><th>95% Interval</th><th>Cases Passed</th><th>Scored Attempts</th><th>Lift</th>
</tr></thead>
<tbody>
{% for name, agent in t3_agents.items() %}
@@ -2589,11 +2591,15 @@
{% set wp = pak.with_skill or {} %}
{% set bp = pak.without_skill or {} %}
{% set lf = pak.lift or {} %}
+ {% set wi = wp.rate_interval or {} %}
+ {% set bi = bp.rate_interval or {} %}
+ {% set paired = lf.paired_comparison or {} %}
{% if wp %}
<tr>
<td><strong>{{ name }}</strong></td>
<td>With skill</td>
<td>{{ "%.0f%%" | format((wp.rate or 0.0) * 100) }}</td>
+ <td>{% if wi %}{{ wi | adaptive_interval_percent }}{% else %}—{% endif %}</td>
<td>{{ wp.passed_cases or 0 }} of {{ wp.total_cases or 0 }}</td>
<td>{{ wp.attempts_used or 0 }} of {{ wp.max_attempts_possible or 0 }}</td>
<td style="color:{{ 'var(--success-color)' if (lf.delta or 0) >= tier3_lift_pass_threshold else ('var(--danger-color)' if (lf.delta or 0) <= tier3_lift_fail_threshold else 'var(--text-secondary)') }};font-weight:600;">
@@ -2606,15 +2612,44 @@
<td></td>
<td>Without skill</td>
<td>{{ "%.0f%%" | format((bp.rate or 0.0) * 100) }}</td>
+ <td>{% if bi %}{{ bi | adaptive_interval_percent }}{% else %}—{% endif %}</td>
<td>{{ bp.passed_cases or 0 }} of {{ bp.total_cases or 0 }}</td>
<td>{{ bp.attempts_used or 0 }} of {{ bp.max_attempts_possible or 0 }}</td>
<td>baseline</td>
</tr>
{% endif %}
+ {% if paired %}
+ <tr>
+ <td></td>
+ <td colspan="6" style="color:var(--text-secondary);font-size:0.82rem;">
+ {% if paired.paired_cases %}
+ Paired cases: {{ paired.paired_cases or 0 }}
+ (skill-only passes {{ paired.with_skill_only_pass or 0 }},
+ baseline-only passes {{ paired.without_skill_only_pass or 0 }},
+ both pass {{ paired.both_pass or 0 }}, neither pass {{ paired.neither_pass or 0 }}).
+ Pairing: {{ paired.pairing_status or 'unavailable' }}.
+ {% if paired.paired_rate_delta is defined and paired.paired_rate_delta is not none %}
+ Paired pass-rate delta: {{ paired.paired_rate_delta | adaptive_percent(true) }}.
+ {% endif %}
+ {% if paired.mcnemar_exact %}
+ Exact McNemar p={% if paired.mcnemar_exact.p_value_text is defined %}{{ paired.mcnemar_exact.p_value_text }}{% else %}{{ "%.4g" | format(paired.mcnemar_exact.p_value) }}{% endif %}.
+ {% if paired.mcnemar_exact.minimum_attainable_p_value is defined %}
+ Minimum attainable p with {{ paired.discordant_cases or 0 }} observed discordant pair{{ 's' if (paired.discordant_cases or 0) != 1 else '' }}: {% if paired.mcnemar_exact.minimum_attainable_p_value_text is defined %}{{ paired.mcnemar_exact.minimum_attainable_p_value_text }}{% else %}{{ "%.4g" | format(paired.mcnemar_exact.minimum_attainable_p_value) }}{% endif %}{% if paired.mcnemar_exact.resolution_limited_at_alpha_0_05 %}
+ (resolution-limited at α=0.05){% endif %}.
+ {% endif %}
+ Diagnostic only.
+ {% endif %}
+ {% else %}
+ Paired comparison {{ paired.pairing_status or 'unavailable' }}; exact test not computed.
+ {% endif %}
+ </td>
+ </tr>
+ {% endif %}
{% endfor %}
</tbody>
</table>
</div>
+ </div>
</details>
{% endif %}
diff --git a/src/skillevaluator/tier3/harbor/collector.py b/src/skillevaluator/tier3/harbor/collector.py
index bdf96e58..c3aa6738 100644
--- a/src/skillevaluator/tier3/harbor/collector.py
+++ b/src/skillevaluator/tier3/harbor/collector.py
@@ -11,10 +11,13 @@ from __future__ import annotations
import copy
import json
import logging
+import math
import os
import re
import shutil
import stat
+import sys
+from fractions import Fraction
from pathlib import Path
from typing import Any
@@ -1941,6 +1944,265 @@ def harbor_job_passed(job_dir: Path, pass_threshold: float) -> bool:
)
+def _wilson_score_interval(successes: int, total: int) -> dict[str, Any] | None:
+ """Return a two-sided 95% Wilson score interval for a binomial rate."""
+ if total <= 0 or successes < 0 or successes > total:
+ return None
+
+ # Standard-normal 97.5th percentile for a two-sided 95% interval.
+ z = 1.959963984540054
+ proportion = successes / total
+ z_squared = z * z
+ denominator = 1.0 + z_squared / total
+ center = (proportion + z_squared / (2.0 * total)) / denominator
+ margin = z * math.sqrt((proportion * (1.0 - proportion) + z_squared / (4.0 * total)) / total) / denominator
+ return {
+ "method": "wilson_score",
+ "confidence_level": 0.95,
+ # At the mathematical endpoints, ``center +/- margin`` can leave a
+ # one-ulp residual on the wrong side of the observed proportion. Pin
+ # those endpoints exactly so the reported interval contains 0 and 1.
+ "lower": 0.0 if successes == 0 else max(0.0, center - margin),
+ "upper": 1.0 if successes == total else min(1.0, center + margin),
+ }
+
+
+def _mcnemar_exact_probability(with_only_pass: int, without_only_pass: int) -> Fraction:
+ """Return the exact two-sided McNemar probability as a rational number."""
+ discordant = with_only_pass + without_only_pass
+ if discordant == 0:
+ return Fraction(1, 1)
+
+ tail = min(with_only_pass, without_only_pass)
+ # Either balanced split covers at least half of the symmetric binomial
+ # distribution, so the doubled tail is exactly one after clamping. This
+ # also avoids constructing thousands of huge coefficients for an obvious
+ # result such as 10,000 versus 10,000 discordant outcomes.
+ if tail == discordant // 2:
+ return Fraction(1, 1)
+
+ coefficient = 1
+ lower_tail = 1
+ for count in range(1, tail + 1):
+ coefficient = coefficient * (discordant - count + 1) // count
+ lower_tail += coefficient
+ return min(Fraction(1, 1), Fraction(2 * lower_tail, 1 << discordant))
+
+
+def _probability_float(probability: Fraction) -> float | None:
+ """Return a nonzero float approximation, or None when conversion underflows."""
+ approximate = float(probability)
+ return approximate if approximate > 0.0 or probability == 0 else None
+
+
+def _probability_text(probability: Fraction) -> str:
+ """Return a bounded nonzero decimal representation for an exact probability."""
+ approximate = _probability_float(probability)
+ # Positive subnormal floats have too few significant bits for the requested
+ # decimal precision, so format them from the exact ratio below as well.
+ if approximate is not None and (probability == 0 or approximate >= sys.float_info.min):
+ return format(approximate, ".10g")
+
+ # ``Decimal(numerator) / Decimal(denominator)`` both scales with the full
+ # integer width and underflows at Decimal's default Emin. For probabilities
+ # already below binary-float range, derive a scientific mantissa from the
+ # integer logarithms instead. The work and output size are bounded by the
+ # operands' bit lengths, and a nonzero Fraction can never be rendered as 0.
+ log10_probability = (math.log2(probability.numerator) - math.log2(probability.denominator)) * math.log10(2.0)
+ exponent = math.floor(log10_probability)
+ mantissa = 10.0 ** (log10_probability - exponent)
+ mantissa_text = format(mantissa, ".10g")
+ if mantissa_text == "10":
+ mantissa_text = "1"
+ exponent += 1
+ return f"{mantissa_text}e{exponent:+d}"
+
+
+_MAX_EXACT_PROBABILITY_INTEGER_DIGITS = 4_300
+_MAX_UNPAIRED_CASE_ID_SAMPLE = 64
+
+
+def _decimal_digit_count(value: int) -> int:
+ """Count base-10 digits without invoking CPython's guarded int-to-string path."""
+ value = abs(value)
+ if value == 0:
+ return 1
+
+ estimate = max(1, int((value.bit_length() - 1) * math.log10(2)) + 1)
+ lower_bound = 10 ** (estimate - 1)
+ while value < lower_bound:
+ estimate -= 1
+ lower_bound //= 10
+ while value >= lower_bound * 10:
+ estimate += 1
+ lower_bound *= 10
+ return estimate
+
+
+def _probability_exact(probability: Fraction) -> str | None:
+ """Return a bounded reduced rational, or None when decimal rendering is unsafe."""
+ active_limit = sys.get_int_max_str_digits()
+ render_limit = (
+ min(_MAX_EXACT_PROBABILITY_INTEGER_DIGITS, active_limit)
+ if active_limit > 0
+ else _MAX_EXACT_PROBABILITY_INTEGER_DIGITS
+ )
+ if max(_decimal_digit_count(probability.numerator), _decimal_digit_count(probability.denominator)) > (render_limit):
+ return None
+ try:
+ if probability.denominator == 1:
+ return str(probability.numerator)
+ return f"{probability.numerator}/{probability.denominator}"
+ except ValueError:
+ # The interpreter-wide limit can change between the digit check and
+ # rendering. Treat that race like any other bounded omission.
+ return None
+
+
+def _exact_probability_fields(field: str, probability: Fraction) -> dict[str, Any]:
+ """Serialize exact probability diagnostics without changing process-wide safety limits."""
+ exact = _probability_exact(probability)
+ fields: dict[str, Any] = {field: exact, f"{field}_omitted": exact is None}
+ if exact is None:
+ fields[f"{field}_omitted_reason"] = "decimal_digit_limit"
+ return fields
+
+
+def _mcnemar_exact_p_value(with_only_pass: int, without_only_pass: int) -> float | None:
+ """Return the two-sided exact McNemar p-value, or None on float underflow."""
+ return _probability_float(_mcnemar_exact_probability(with_only_pass, without_only_pass))
+
+
+def _minimum_attainable_mcnemar_probability(discordant: int) -> Fraction:
+ """Return the smallest two-sided exact probability attainable for this pair count."""
+ if discordant <= 1:
+ return Fraction(1, 1)
+ return Fraction(2, 1 << discordant)
+
+
+def _minimum_attainable_mcnemar_p_value(discordant: int) -> float | None:
+ """Return the smallest two-sided exact p-value attainable for this pair count."""
+ return _probability_float(_minimum_attainable_mcnemar_probability(discordant))
+
+
+def _pass_rate_delta(with_skill: dict[str, Any], without_skill: dict[str, Any]) -> float:
+ """Return the legacy delta contract derived from persisted rounded rates."""
+ return round(float(with_skill.get("rate", 0.0) or 0.0) - float(without_skill.get("rate", 0.0) or 0.0), 4)
+
+
+def _count_derived_pass_rate_delta(with_skill: dict[str, Any], without_skill: dict[str, Any]) -> float:
+ """Return the corrected arm-level delta derived directly from pass counts."""
+ with_total = int(with_skill.get("total_cases", 0) or 0)
+ without_total = int(without_skill.get("total_cases", 0) or 0)
+ if with_total > 0 and without_total > 0:
+ with_rate = int(with_skill.get("passed_cases", 0) or 0) / with_total
+ without_rate = int(without_skill.get("passed_cases", 0) or 0) / without_total
+ return round(with_rate - without_rate, 4)
+ return _pass_rate_delta(with_skill, without_skill)
+
+
+def _paired_pass_comparison(
+ with_skill: dict[str, Any],
+ without_skill: dict[str, Any],
+) -> dict[str, Any]:
+ """Compare pass@k outcomes for matching cases across both evaluation arms."""
+ with_cases = with_skill.get("cases")
+ without_cases = without_skill.get("cases")
+ if not isinstance(with_cases, dict) or not isinstance(without_cases, dict):
+ return {"pairing_status": "unavailable", "paired_cases": 0}
+
+ with_expected = {str(case_id): case for case_id, case in with_cases.items() if not case.get("extra_case")}
+ without_expected = {str(case_id): case for case_id, case in without_cases.items() if not case.get("extra_case")}
+ common_ids = sorted(set(with_expected) & set(without_expected))
+ with_only_ids = sorted(set(with_expected) - set(without_expected))
+ without_only_ids = sorted(set(without_expected) - set(with_expected))
+
+ with_total = int(with_skill.get("total_cases", 0) or 0)
+ without_total = int(without_skill.get("total_cases", 0) or 0)
+ unidentified_with = max(0, with_total - len(with_expected))
+ unidentified_without = max(0, without_total - len(without_expected))
+ complete = (
+ bool(common_ids)
+ and not with_only_ids
+ and not without_only_ids
+ and unidentified_with == 0
+ and unidentified_without == 0
+ and len(common_ids) == with_total == without_total
+ )
+
+ outcomes = {
+ "both_pass": 0,
+ "with_skill_only_pass": 0,
+ "without_skill_only_pass": 0,
+ "neither_pass": 0,
+ }
+ for case_id in common_ids:
+ with_passed = bool(with_expected[case_id].get("passed"))
+ without_passed = bool(without_expected[case_id].get("passed"))
+ if with_passed and without_passed:
+ outcomes["both_pass"] += 1
+ elif with_passed:
+ outcomes["with_skill_only_pass"] += 1
+ elif without_passed:
+ outcomes["without_skill_only_pass"] += 1
+ else:
+ outcomes["neither_pass"] += 1
+
+ paired_cases = len(common_ids)
+ paired_delta = (
+ (outcomes["with_skill_only_pass"] - outcomes["without_skill_only_pass"]) / paired_cases
+ if paired_cases
+ else None
+ )
+ result: dict[str, Any] = {
+ "pairing_status": "complete" if complete else ("partial" if common_ids else "unavailable"),
+ "paired_cases": paired_cases,
+ "with_skill_unpaired_case_count": len(with_only_ids),
+ "without_skill_unpaired_case_count": len(without_only_ids),
+ "with_skill_unpaired_case_ids": with_only_ids[:_MAX_UNPAIRED_CASE_ID_SAMPLE],
+ "without_skill_unpaired_case_ids": without_only_ids[:_MAX_UNPAIRED_CASE_ID_SAMPLE],
+ "with_skill_unpaired_case_ids_truncated": len(with_only_ids) > _MAX_UNPAIRED_CASE_ID_SAMPLE,
+ "without_skill_unpaired_case_ids_truncated": len(without_only_ids) > _MAX_UNPAIRED_CASE_ID_SAMPLE,
+ "with_skill_unidentified_cases": unidentified_with,
+ "without_skill_unidentified_cases": unidentified_without,
+ **outcomes,
+ "discordant_cases": outcomes["with_skill_only_pass"] + outcomes["without_skill_only_pass"],
+ "paired_rate_delta": paired_delta,
+ }
+ if complete:
+ discordant = result["discordant_cases"]
+ exact_probability = _mcnemar_exact_probability(
+ outcomes["with_skill_only_pass"],
+ outcomes["without_skill_only_pass"],
+ )
+ minimum_attainable_probability = _minimum_attainable_mcnemar_probability(discordant)
+ exact_probability_float = _probability_float(exact_probability)
+ minimum_attainable_float = _probability_float(minimum_attainable_probability)
+ exact_probability_text = _probability_text(exact_probability)
+ minimum_attainable_text = (
+ exact_probability_text
+ if minimum_attainable_probability == exact_probability
+ else _probability_text(minimum_attainable_probability)
+ )
+ result["mcnemar_exact"] = {
+ "method": "two_sided_exact_binomial",
+ "null_hypothesis": "equal marginal pass probabilities",
+ "p_value": exact_probability_float,
+ "p_value_text": exact_probability_text,
+ **_exact_probability_fields("p_value_exact", exact_probability),
+ "p_value_numeric_underflow": exact_probability_float is None,
+ "minimum_attainable_p_value": minimum_attainable_float,
+ "minimum_attainable_p_value_text": minimum_attainable_text,
+ **_exact_probability_fields(
+ "minimum_attainable_p_value_exact",
+ minimum_attainable_probability,
+ ),
+ "minimum_attainable_p_value_numeric_underflow": minimum_attainable_float is None,
+ "resolution_limited_at_alpha_0_05": minimum_attainable_probability > Fraction(1, 20),
+ }
+ return result
+
+
def _pass_summary(
rewards: list[dict[str, Any]],
*,
@@ -2022,6 +2284,7 @@ def _pass_summary(
total_cases = len(grouped)
failed_cases = max(0, total_cases - passed_cases)
rate = round(passed_cases / total_cases, 4) if total_cases else 0.0
+ rate_interval = _wilson_score_interval(passed_cases, total_cases)
return {
"k": n_attempts,
@@ -2031,6 +2294,7 @@ def _pass_summary(
"failed_cases": failed_cases,
"total_cases": total_cases,
"rate": rate,
+ "rate_interval": rate_interval,
"attempts_used": attempts_used,
"max_attempts_possible": total_cases * n_attempts,
"avg_attempts_used": round(attempts_used / total_cases, 4) if total_cases else 0.0,
@@ -3145,8 +3409,10 @@ def collect_harbor_results(
pass_lift = {
"with_skill": with_pass.get("rate", 0.0),
"without_skill": without_pass.get("rate", 0.0),
- "delta": round(with_pass.get("rate", 0.0) - without_pass.get("rate", 0.0), 4),
+ "delta": _pass_rate_delta(with_pass, without_pass),
+ "count_derived_delta": _count_derived_pass_rate_delta(with_pass, without_pass),
"passed_cases_delta": int(with_pass.get("passed_cases", 0)) - int(without_pass.get("passed_cases", 0)),
+ "paired_comparison": _paired_pass_comparison(with_pass, without_pass),
}
(agent_dir / "pass_at_k_lift.json").write_text(json.dumps(pass_lift, indent=2), encoding="utf-8")
diff --git a/tests/reporting/test_unified_tier3_report.py b/tests/reporting/test_unified_tier3_report.py
index 4e2854cf..832d0f84 100644
--- a/tests/reporting/test_unified_tier3_report.py
+++ b/tests/reporting/test_unified_tier3_report.py
@@ -28,6 +28,7 @@ from skillevaluator.models import ValidationResult
from skillevaluator.reporting import HTMLReporter, JSONReporter
from skillevaluator.reporting import html as html_module
from skillevaluator.reporting.html import PackageLoader, _compact_json
+from skillevaluator.tier3.harbor.collector import _paired_pass_comparison, _wilson_score_interval
from skillevaluator.tier3.harbor.metrics import DEFAULT_METRICS
from skillevaluator.tier3.harbor.report_data import build_dataset_snapshot
@@ -985,6 +986,85 @@ def test_canonical_payload_enforces_total_serialized_budget() -> None:
assert len(encoded) <= truncation["payload_budget_bytes"]
+def test_unpaired_id_artifact_survives_loader_report_budget_and_html(tmp_path: Path) -> None:
+ skill = tmp_path / "demo"
+ skill.mkdir()
+ run_dir = tmp_path / "results" / "20260825_120000"
+ agent_dir = run_dir / "opencode"
+ case_count = 2_400
+
+ for variant, rate, passed_cases in (("with-skill", 1.0, 1), ("without-skill", 0.0, 0)):
+ summary_path = agent_dir / variant / "summary.json"
+ summary_path.parent.mkdir(parents=True)
+ score = rate
+ summary_path.write_text(
+ json.dumps(
+ {
+ "scores": dict.fromkeys(DEFAULT_METRICS, score),
+ "metrics": list(DEFAULT_METRICS),
+ "num_trials": 1,
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ "overall_score": score,
+ "pass_at_k": {
+ "rate": rate,
+ "passed_cases": passed_cases,
+ "total_cases": 1,
+ "attempts_used": 1,
+ "max_attempts_possible": 1,
+ },
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ full_with_ids = [f"with-{index:04d}" for index in range(case_count)]
+ full_without_ids = [f"without-{index:04d}" for index in range(case_count)]
+ pass_lift_path = agent_dir / "pass_at_k_lift.json"
+ pass_lift_path.write_text(
+ json.dumps(
+ {
+ "with_skill": 1.0,
+ "without_skill": 0.0,
+ "delta": 1.0,
+ "paired_comparison": {
+ "pairing_status": "unavailable",
+ "paired_cases": 0,
+ "with_skill_unpaired_case_ids": full_with_ids,
+ "without_skill_unpaired_case_ids": full_without_ids,
+ },
+ }
+ ),
+ encoding="utf-8",
+ )
+ assert pass_lift_path.stat().st_size < 2 * 1024 * 1024
+
+ result = agent_eval_result_from_directory(skill, run_dir, use_llm_judge=False)
+
+ assert result is not None
+ payload = result.metadata["agent_eval"]
+ paired = payload["agents"]["opencode"]["pass_at_k"]["lift"]["paired_comparison"]
+ assert payload["agents"]
+ assert payload["pass_at_k"]
+ assert paired["with_skill_unpaired_case_count"] == case_count
+ assert paired["without_skill_unpaired_case_count"] == case_count
+ assert len(paired["with_skill_unpaired_case_ids"]) == 64
+ assert len(paired["without_skill_unpaired_case_ids"]) == 64
+ assert paired["with_skill_unpaired_case_ids_truncated"] is True
+ assert paired["without_skill_unpaired_case_ids_truncated"] is True
+ # The canonical payload intentionally carries both a best-agent projection
+ # and the top-level pass@k projection, so the omission signal counts both
+ # serialized copies that were bounded.
+ assert payload["report_truncation"]["omitted"]["unpaired_case_ids"] == 4 * (case_count - 64)
+ assert len(json.dumps(payload, separators=(",", ":")).encode()) <= 2 * 1024 * 1024
+
+ html = HTMLReporter(include_timestamp=False).render_all([result])
+ assert "Paired comparison unavailable; exact test not computed." in html
+ assert "Pass@" in html
+
+
def test_standalone_tier3_persists_the_canonical_feedback_payload(tmp_path: Path) -> None:
skill = tmp_path / "demo"
skill.mkdir()
@@ -1171,3 +1251,323 @@ def test_partial_pass_at_k_case_without_attempts_skipped_still_renders(tmp_path:
report = render_agent_eval_html_report(skill, run_dir, use_llm_judge=False)
assert "Attempt Details" in report.read_text(encoding="utf-8")
+
+
+def test_pass_at_k_uncertainty_and_paired_evidence_render() -> None:
+ payload = build_agent_eval_payload(
+ "paired-evidence-demo",
+ {
+ "codex": {
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 8,
+ "scored_attempts": 8,
+ "overall_with_skill": 0.75,
+ "overall_without_skill": 0.5,
+ "pass_with_skill": {
+ "rate": 0.75,
+ "rate_interval": {"lower": 0.3006, "upper": 0.9544},
+ "passed_cases": 3,
+ "total_cases": 4,
+ "attempts_used": 4,
+ "max_attempts_possible": 4,
+ },
+ "pass_without_skill": {
+ "rate": 0.5,
+ "rate_interval": {"lower": 0.15, "upper": 0.85},
+ "passed_cases": 2,
+ "total_cases": 4,
+ "attempts_used": 4,
+ "max_attempts_possible": 4,
+ },
+ "pass_lift": {
+ "delta": 0.25,
+ "paired_comparison": {
+ "pairing_status": "complete",
+ "paired_cases": 4,
+ "with_skill_only_pass": 1,
+ "without_skill_only_pass": 0,
+ "both_pass": 2,
+ "neither_pass": 1,
+ "discordant_cases": 1,
+ "paired_rate_delta": 0.25,
+ "mcnemar_exact": {
+ "p_value": 1.0,
+ "p_value_text": "1",
+ "minimum_attainable_p_value": 1.0,
+ "minimum_attainable_p_value_text": "1",
+ "resolution_limited_at_alpha_0_05": True,
+ },
+ },
+ },
+ }
+ },
+ attempt_policy={"max_attempts": 1, "pass_threshold": 0.5},
+ use_llm_judge=False,
+ )
+ assert payload is not None
+
+ html = _render_agent_payload(payload)
+
+ assert "30%\u201395%" in html
+ assert "15%\u201385%" in html
+ assert "skill-only passes 1" in html
+ assert "baseline-only passes 0" in html
+ assert "Pairing: complete" in html
+ assert "Paired pass-rate delta: +25.0%" in html
+ assert "Exact McNemar p=1" in html
+ assert "Minimum attainable p with 1 observed discordant pair:" in html
+ assert "resolution-limited at \N{GREEK SMALL LETTER ALPHA}=0.05" in html
+
+
+def test_pass_at_k_render_preserves_exact_subnormal_mcnemar_probability() -> None:
+ pair_count = 1_086
+ case_ids = [f"case-{index}" for index in range(pair_count)]
+ with_skill_cases = {case_id: {"passed": index < 1_085} for index, case_id in enumerate(case_ids)}
+ without_skill_cases = {case_id: {"passed": index == 1_085} for index, case_id in enumerate(case_ids)}
+ paired = _paired_pass_comparison(
+ {"total_cases": pair_count, "cases": with_skill_cases},
+ {"total_cases": pair_count, "cases": without_skill_cases},
+ )
+
+ payload = build_agent_eval_payload(
+ "subnormal-mcnemar-probability-demo",
+ {
+ "codex": {
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": pair_count * 2,
+ "scored_attempts": pair_count * 2,
+ "overall_with_skill": 1.0,
+ "overall_without_skill": 0.0,
+ "pass_with_skill": {
+ "rate": 1_085 / pair_count,
+ "passed_cases": 1_085,
+ "total_cases": pair_count,
+ },
+ "pass_without_skill": {
+ "rate": 1 / pair_count,
+ "passed_cases": 1,
+ "total_cases": pair_count,
+ },
+ "pass_lift": {
+ "delta": 1_084 / pair_count,
+ "paired_comparison": paired,
+ },
+ }
+ },
+ attempt_policy={"max_attempts": 1, "pass_threshold": 0.5},
+ use_llm_judge=False,
+ )
+ assert payload is not None
+
+ html = _render_agent_payload(payload)
+
+ assert paired["mcnemar_exact"]["p_value_text"] == "2.622311314e-324"
+ assert "Exact McNemar p=2.622311314e-324." in html
+ assert "4.940656458e-324" not in html
+
+
+def test_pass_at_k_render_preserves_narrow_intervals_and_small_paired_delta() -> None:
+ payload = build_agent_eval_payload(
+ "adaptive-pass-evidence-demo",
+ {
+ "codex": {
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 20_000,
+ "scored_attempts": 20_000,
+ "overall_with_skill": 0.5,
+ "overall_without_skill": 0.5,
+ "pass_with_skill": {
+ "rate": 0.0,
+ "rate_interval": {"lower": 0.0, "upper": 0.0038},
+ "passed_cases": 0,
+ "total_cases": 1_000,
+ },
+ "pass_without_skill": {
+ "rate": 1.0,
+ "rate_interval": {"lower": 0.9962, "upper": 1.0},
+ "passed_cases": 1_000,
+ "total_cases": 1_000,
+ },
+ "pass_lift": {
+ "delta": -1.0,
+ "paired_comparison": {
+ "pairing_status": "complete",
+ "paired_cases": 10_000,
+ "with_skill_only_pass": 1,
+ "without_skill_only_pass": 0,
+ "both_pass": 0,
+ "neither_pass": 9_999,
+ "discordant_cases": 1,
+ "paired_rate_delta": 0.0001,
+ },
+ },
+ }
+ },
+ attempt_policy={"max_attempts": 1, "pass_threshold": 0.5},
+ use_llm_judge=False,
+ )
+ assert payload is not None
+
+ html = _render_agent_payload(payload)
+
+ assert "0%\u20130.38%" in html
+ assert "99.62%\u2013100%" in html
+ assert "Paired pass-rate delta: +0.01%" in html
+
+
+def test_pass_at_k_render_preserves_sub_basis_point_endpoint_uncertainty() -> None:
+ payload = build_agent_eval_payload(
+ "endpoint-precision-demo",
+ {
+ "codex": {
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 200_000,
+ "scored_attempts": 200_000,
+ "overall_with_skill": 0.5,
+ "overall_without_skill": 0.5,
+ "pass_with_skill": {
+ "rate": 0.0,
+ "rate_interval": {"lower": 0.0, "upper": 0.000038413},
+ "passed_cases": 0,
+ "total_cases": 100_000,
+ },
+ "pass_without_skill": {
+ "rate": 1.0,
+ "rate_interval": {"lower": 0.999961587, "upper": 1.0},
+ "passed_cases": 100_000,
+ "total_cases": 100_000,
+ },
+ "pass_lift": {
+ "delta": -1.0,
+ "paired_comparison": {
+ "pairing_status": "complete",
+ "paired_cases": 25_000,
+ "with_skill_only_pass": 1,
+ "without_skill_only_pass": 0,
+ "both_pass": 0,
+ "neither_pass": 24_999,
+ "discordant_cases": 1,
+ "paired_rate_delta": 1 / 25_000,
+ },
+ },
+ }
+ },
+ attempt_policy={"max_attempts": 1, "pass_threshold": 0.5},
+ use_llm_judge=False,
+ )
+ assert payload is not None
+
+ html = _render_agent_payload(payload)
+
+ assert "0%\u20130.004%" in html
+ assert "99.996%\u2013100%" in html
+ assert "Paired pass-rate delta: +0.004%" in html
+
+
+def test_pass_at_k_render_preserves_distinct_central_high_n_wilson_bounds() -> None:
+ interval = _wilson_score_interval(50_000, 100_000)
+ assert interval is not None
+ payload = build_agent_eval_payload(
+ "central-interval-precision-demo",
+ {
+ "codex": {
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 200_000,
+ "scored_attempts": 200_000,
+ "overall_with_skill": 0.5,
+ "overall_without_skill": 0.5,
+ "pass_with_skill": {
+ "rate": 0.5,
+ "rate_interval": interval,
+ "passed_cases": 50_000,
+ "total_cases": 100_000,
+ },
+ "pass_without_skill": {
+ "rate": 0.5,
+ "rate_interval": interval,
+ "passed_cases": 50_000,
+ "total_cases": 100_000,
+ },
+ "pass_lift": {"delta": 0.0},
+ }
+ },
+ attempt_policy={"max_attempts": 1, "pass_threshold": 0.5},
+ use_llm_judge=False,
+ )
+ assert payload is not None
+
+ html = _render_agent_payload(payload)
+
+ assert "49.7%\u201350.3%" in html
+ assert "50%\u201350%" not in html
+
+
+def test_pass_at_k_final_column_has_a_narrow_viewport_scroll_contract() -> None:
+ payload = build_agent_eval_payload(
+ "responsive-pass-evidence-demo",
+ {
+ "codex": {
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 2,
+ "scored_attempts": 2,
+ "overall_with_skill": 1.0,
+ "overall_without_skill": 0.0,
+ "pass_with_skill": {"rate": 1.0, "passed_cases": 1, "total_cases": 1},
+ "pass_without_skill": {"rate": 0.0, "passed_cases": 0, "total_cases": 1},
+ "pass_lift": {"delta": 1.0},
+ }
+ },
+ attempt_policy={"max_attempts": 1, "pass_threshold": 0.5},
+ use_llm_judge=False,
+ )
+ assert payload is not None
+
+ html = _render_agent_payload(payload)
+ region = re.search(
+ r'<div class="table-scroll" tabindex="0" role="region" '
+ r'aria-label="Pass at K summary table">(?P<table>.*?)</div>',
+ html,
+ re.DOTALL,
+ )
+
+ assert region is not None
+ assert 'class="summary-table pass-at-k-table"' in region.group("table")
+ assert re.search(r"<th>Lift</th>\s*</tr>", region.group("table"))
+ assert ".table-scroll { max-width: 100%; overflow-x: auto;" in html
+ assert ".pass-at-k-table { min-width: 720px; }" in html
+
+
+def test_unavailable_pairing_does_not_render_a_zero_evidence_row() -> None:
+ payload = build_agent_eval_payload(
+ "unpaired-evidence-demo",
+ {
+ "codex": {
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 2,
+ "scored_attempts": 2,
+ "overall_with_skill": 1.0,
+ "overall_without_skill": 0.0,
+ "pass_with_skill": {"rate": 1.0, "passed_cases": 1, "total_cases": 1},
+ "pass_without_skill": {"rate": 0.0, "passed_cases": 0, "total_cases": 1},
+ "pass_lift": {
+ "delta": 1.0,
+ "paired_comparison": {"pairing_status": "unavailable", "paired_cases": 0},
+ },
+ }
+ },
+ attempt_policy={"max_attempts": 1, "pass_threshold": 0.5},
+ use_llm_judge=False,
+ )
+ assert payload is not None
+
+ html = _render_agent_payload(payload)
+
+ assert "Paired cases: 0" not in html
+ assert "Paired comparison unavailable; exact test not computed." in html
diff --git a/tests/test_harbor_collector_runtime_failures.py b/tests/test_harbor_collector_runtime_failures.py
index d0401b18..0f1a1043 100644
--- a/tests/test_harbor_collector_runtime_failures.py
+++ b/tests/test_harbor_collector_runtime_failures.py
@@ -472,11 +472,12 @@ def _write_reward(
score: float = 0.25,
steps: tuple[str, ...] = (),
trial_name: str | None = None,
+ include_entry_id: bool = True,
+ result_task_name: str | None = None,
) -> None:
trial = jobs_dir / f"demo-opencode-{variant}" / (trial_name or f"{case_id}_attempt{attempt:03d}")
verifier_dirs = [trial / "steps" / step / "verifier" for step in steps] or [trial / "verifier"]
reward = {
- "entry_id": case_id,
"overall": score,
"security": score,
"skill_execution": score,
@@ -485,9 +486,16 @@ def _write_reward(
"goal_accuracy": score,
"behavior_check": score,
}
+ if include_entry_id:
+ reward["entry_id"] = case_id
for verifier_dir in verifier_dirs:
verifier_dir.mkdir(parents=True, exist_ok=True)
(verifier_dir / "reward.json").write_text(json.dumps(reward), encoding="utf-8")
+ if result_task_name is not None:
+ (trial / "result.json").write_text(
+ json.dumps({"trial_name": trial.name, "task_name": result_task_name}),
+ encoding="utf-8",
+ )
def _write_variant_job_results(jobs_dir: Path, variants: tuple[str, ...] = ("with", "without")) -> None:
@@ -529,6 +537,88 @@ def test_stop_on_pass_does_not_report_intentionally_skipped_attempts(tmp_path: P
assert result["scored_attempts"] == 4
+def test_complete_ab_run_records_paired_pass_evidence(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ _write_reward(jobs_dir, variant="with", case_id="case-a", attempt=1, score=1.0)
+ _write_reward(jobs_dir, variant="with", case_id="case-b", attempt=1, score=1.0)
+ _write_reward(jobs_dir, variant="without", case_id="case-a", attempt=1, score=0.0)
+ _write_reward(jobs_dir, variant="without", case_id="case-b", attempt=1, score=1.0)
+ _write_variant_job_results(jobs_dir)
+
+ result = _collect(tmp_path, n_attempts=1)
+
+ assert result["execution_status"] == "succeeded"
+ pass_at_k = result["agents"]["opencode"]["pass_at_k"]
+ assert pass_at_k["with_skill"]["rate_interval"]["confidence_level"] == 0.95
+ paired = pass_at_k["lift"]["paired_comparison"]
+ assert paired["pairing_status"] == "complete"
+ assert paired["paired_cases"] == 2
+ assert paired["with_skill_only_pass"] == 1
+ assert paired["without_skill_only_pass"] == 0
+ assert paired["paired_rate_delta"] == 0.5
+ assert paired["mcnemar_exact"]["p_value"] == 1.0
+
+ persisted = json.loads((tmp_path / "results/opencode/pass_at_k_lift.json").read_text(encoding="utf-8"))
+ assert persisted["delta"] == 0.5
+ assert persisted["count_derived_delta"] == 0.5
+ assert persisted["paired_comparison"] == paired
+
+
+def test_result_derived_case_ids_exercise_partial_pairing_through_collector(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ _write_reward(
+ jobs_dir,
+ variant="with",
+ case_id="unused",
+ attempt=1,
+ score=1.0,
+ trial_name="opaque-with-shared",
+ include_entry_id=False,
+ result_task_name="suite/shared",
+ )
+ _write_reward(
+ jobs_dir,
+ variant="with",
+ case_id="unused",
+ attempt=1,
+ score=1.0,
+ trial_name="opaque-with-only",
+ include_entry_id=False,
+ result_task_name="suite/with-only",
+ )
+ _write_reward(
+ jobs_dir,
+ variant="without",
+ case_id="unused",
+ attempt=1,
+ score=0.0,
+ trial_name="opaque-without-shared",
+ include_entry_id=False,
+ result_task_name="suite/shared",
+ )
+ _write_reward(
+ jobs_dir,
+ variant="without",
+ case_id="unused",
+ attempt=1,
+ score=0.0,
+ trial_name="opaque-without-only",
+ include_entry_id=False,
+ result_task_name="suite/without-only",
+ )
+ _write_variant_job_results(jobs_dir)
+
+ result = _collect(tmp_path, n_attempts=1, expected_cases=2, expected_case_ids=None)
+
+ assert result["execution_status"] == "succeeded"
+ paired = result["agents"]["opencode"]["pass_at_k"]["lift"]["paired_comparison"]
+ assert paired["pairing_status"] == "partial"
+ assert paired["paired_cases"] == 1
+ assert paired["with_skill_unpaired_case_ids"] == ["with-only"]
+ assert paired["without_skill_unpaired_case_ids"] == ["without-only"]
+ assert "mcnemar_exact" not in paired
+
+
def test_stop_on_pass_records_skipped_attempts_in_pass_summary(tmp_path: Path) -> None:
jobs_dir = tmp_path / "jobs"
_write_reward(jobs_dir, variant="with", case_id="case-a", attempt=1, score=0.2)
diff --git a/tests/test_harbor_metrics_truth.py b/tests/test_harbor_metrics_truth.py
index a27a2412..eaf33479 100644
--- a/tests/test_harbor_metrics_truth.py
+++ b/tests/test_harbor_metrics_truth.py
@@ -4,10 +4,28 @@
from __future__ import annotations
import math
+import os
+import subprocess
+import sys
+import time
+from fractions import Fraction
import pytest
-from skillevaluator.tier3.harbor.collector import _compute_lift, _condition_execution_summary, _pass_summary
+from skillevaluator.tier3.harbor import collector as collector_module
+from skillevaluator.tier3.harbor.collector import (
+ _compute_lift,
+ _condition_execution_summary,
+ _count_derived_pass_rate_delta,
+ _mcnemar_exact_p_value,
+ _mcnemar_exact_probability,
+ _minimum_attainable_mcnemar_p_value,
+ _paired_pass_comparison,
+ _pass_rate_delta,
+ _pass_summary,
+ _probability_text,
+ _wilson_score_interval,
+)
from skillevaluator.tier3.harbor.metrics import (
CUSTOM_ONLY_METRIC_SET,
DEFAULT_METRIC_SET,
@@ -99,6 +117,338 @@ def test_pass_summary_marks_incomplete_reward_unscored() -> None:
assert math.isfinite(summary["rate"])
+ ("successes", "total", "expected"),
+ [
+ (0, 4, (0.0, 0.4899)),
+ (2, 4, (0.15, 0.85)),
+ (4, 4, (0.5101, 1.0)),
+ ],
+)
+def test_wilson_interval_reports_bounded_case_rate_uncertainty(
+ successes: int,
+ total: int,
+ expected: tuple[float, float],
+) -> None:
+ interval = _wilson_score_interval(successes, total)
+
+ assert interval is not None
+ assert interval["method"] == "wilson_score"
+ assert interval["confidence_level"] == 0.95
+ assert interval["lower"] == pytest.approx(expected[0], abs=0.0001)
+ assert interval["upper"] == pytest.approx(expected[1], abs=0.0001)
+
+
+def test_wilson_interval_is_unavailable_without_cases() -> None:
+ assert _wilson_score_interval(0, 0) is None
+
+
[email protected]("total", [1, 2, 3, 10, 1_000, 100_000])
+def test_wilson_interval_contains_observed_mathematical_endpoints_exactly(total: int) -> None:
+ zero_passes = _wilson_score_interval(0, total)
+ all_passes = _wilson_score_interval(total, total)
+
+ assert zero_passes is not None
+ assert all_passes is not None
+ assert zero_passes["lower"] == 0.0
+ assert zero_passes["lower"] <= 0.0 <= zero_passes["upper"]
+ assert all_passes["upper"] == 1.0
+ assert all_passes["lower"] <= 1.0 <= all_passes["upper"]
+
+
+def test_wilson_interval_preserves_endpoint_uncertainty_beyond_four_decimal_places() -> None:
+ zero_passes = _wilson_score_interval(0, 100_000)
+ all_passes = _wilson_score_interval(100_000, 100_000)
+
+ assert zero_passes is not None
+ assert all_passes is not None
+ assert zero_passes["lower"] == 0.0
+ assert 0.0 < zero_passes["upper"] < 0.0001
+ assert 0.9999 < all_passes["lower"] < 1.0
+ assert all_passes["upper"] == 1.0
+
+
+def test_mcnemar_exact_p_value_preserves_small_nonzero_results() -> None:
+ assert _mcnemar_exact_p_value(32, 0) == pytest.approx(2**-31)
+ assert _mcnemar_exact_p_value(32, 0) > 0.0
+
+
+ ("discordant", "expected"),
+ [(1, 1.0), (2, 0.5), (3, 0.25), (4, 0.125), (5, 0.0625), (6, 0.03125)],
+)
+def test_mcnemar_exact_reports_attainable_resolution(discordant: int, expected: float) -> None:
+ assert _minimum_attainable_mcnemar_p_value(discordant) == expected
+
+
+def test_pass_rate_delta_preserves_legacy_rate_contract_and_exposes_count_correction() -> None:
+ with_skill = {"passed_cases": 2, "total_cases": 3, "rate": 0.6667}
+ without_skill = {"passed_cases": 1, "total_cases": 3, "rate": 0.3333}
+
+ assert _pass_rate_delta(with_skill, without_skill) == 0.3334
+ assert _count_derived_pass_rate_delta(with_skill, without_skill) == 0.3333
+
+
+ ("with_skill", "without_skill", "legacy_delta", "count_derived_delta"),
+ [
+ (
+ {"passed_cases": 17, "total_cases": 160, "rate": 0.1062},
+ {"passed_cases": 9, "total_cases": 160, "rate": 0.0563},
+ 0.0499,
+ 0.05,
+ ),
+ (
+ {"passed_cases": 1, "total_cases": 160, "rate": 0.0063},
+ {"passed_cases": 17, "total_cases": 160, "rate": 0.1062},
+ -0.0999,
+ -0.1,
+ ),
+ ],
+)
+def test_pass_rate_delta_threshold_compatibility(
+ with_skill: dict[str, float | int],
+ without_skill: dict[str, float | int],
+ legacy_delta: float,
+ count_derived_delta: float,
+) -> None:
+ assert _pass_rate_delta(with_skill, without_skill) == legacy_delta
+ assert _count_derived_pass_rate_delta(with_skill, without_skill) == count_derived_delta
+
+
+def test_balanced_large_exact_test_uses_constant_time_symmetric_tail() -> None:
+ started = time.perf_counter()
+
+ probability = _mcnemar_exact_probability(10_000, 10_000)
+
+ assert probability == 1
+ assert time.perf_counter() - started < 2.0
+
+
+def test_paired_pass_comparison_preserves_case_direction_and_exact_test() -> None:
+ with_skill = {
+ "total_cases": 4,
+ "cases": {
+ "both": {"passed": True},
+ "improved": {"passed": True},
+ "regressed": {"passed": False},
+ "neither": {"passed": False},
+ },
+ }
+ without_skill = {
+ "total_cases": 4,
+ "cases": {
+ "both": {"passed": True},
+ "improved": {"passed": False},
+ "regressed": {"passed": True},
+ "neither": {"passed": False},
+ },
+ }
+
+ paired = _paired_pass_comparison(with_skill, without_skill)
+
+ assert paired["pairing_status"] == "complete"
+ assert paired["paired_cases"] == 4
+ assert paired["both_pass"] == 1
+ assert paired["with_skill_only_pass"] == 1
+ assert paired["without_skill_only_pass"] == 1
+ assert paired["neither_pass"] == 1
+ assert paired["paired_rate_delta"] == 0.0
+ assert paired["mcnemar_exact"]["p_value"] == 1.0
+ assert paired["mcnemar_exact"]["minimum_attainable_p_value"] == 0.5
+ assert paired["mcnemar_exact"]["resolution_limited_at_alpha_0_05"] is True
+
+
+def test_complete_pairing_delta_matches_count_derived_arm_delta() -> None:
+ with_skill = {
+ "passed_cases": 2,
+ "total_cases": 3,
+ "rate": 0.6667,
+ "cases": {"a": {"passed": True}, "b": {"passed": True}, "c": {"passed": False}},
+ }
+ without_skill = {
+ "passed_cases": 1,
+ "total_cases": 3,
+ "rate": 0.3333,
+ "cases": {"a": {"passed": False}, "b": {"passed": True}, "c": {"passed": False}},
+ }
+
+ paired = _paired_pass_comparison(with_skill, without_skill)
+
+ assert paired["paired_rate_delta"] == pytest.approx(1 / 3)
+ assert paired["paired_rate_delta"] == pytest.approx(
+ _count_derived_pass_rate_delta(with_skill, without_skill),
+ abs=0.0001,
+ )
+
+
[email protected](("skill_only", "direction"), [(True, 1), (False, -1)])
+def test_paired_delta_preserves_small_nonzero_direction(skill_only: bool, direction: int) -> None:
+ pair_count = 25_000
+ case_ids = [f"case-{index}" for index in range(pair_count)]
+ with_skill_cases = {case_id: {"passed": False} for case_id in case_ids}
+ without_skill_cases = {case_id: {"passed": False} for case_id in case_ids}
+ (with_skill_cases if skill_only else without_skill_cases)[case_ids[0]]["passed"] = True
+
+ paired = _paired_pass_comparison(
+ {"total_cases": pair_count, "cases": with_skill_cases},
+ {"total_cases": pair_count, "cases": without_skill_cases},
+ )
+
+ assert paired["paired_rate_delta"] == pytest.approx(direction / pair_count)
+ assert math.copysign(1.0, paired["paired_rate_delta"]) == direction
+
+
+def test_mcnemar_exact_preserves_machine_readable_value_beyond_float_range() -> None:
+ with_skill_cases = {f"case-{index}": {"passed": True} for index in range(1076)}
+ without_skill_cases = {case_id: {"passed": False} for case_id in with_skill_cases}
+
+ paired = _paired_pass_comparison(
+ {"total_cases": 1076, "cases": with_skill_cases},
+ {"total_cases": 1076, "cases": without_skill_cases},
+ )
+ exact = paired["mcnemar_exact"]
+
+ assert exact["p_value"] is None
+ assert exact["p_value_numeric_underflow"] is True
+ assert exact["p_value_text"] != "0"
+ assert exact["p_value_exact"].startswith("1/")
+ assert exact["minimum_attainable_p_value"] is None
+ assert exact["minimum_attainable_p_value_text"] != "0"
+ assert exact["minimum_attainable_p_value_exact"] == exact["p_value_exact"]
+
+
+def test_probability_text_is_fast_and_nonzero_beyond_decimal_exponent_range() -> None:
+ probability = collector_module._minimum_attainable_mcnemar_probability(3_321_957)
+
+ started = time.perf_counter()
+ rendered = _probability_text(probability)
+ elapsed = time.perf_counter() - started
+
+ mantissa, separator, exponent = rendered.partition("e")
+ assert separator == "e"
+ assert float(mantissa) > 0.0
+ assert int(exponent) < -1_000_000
+ assert elapsed < 2.0
+
+
+def test_complete_one_sided_pairing_formats_identical_probability_once(monkeypatch: pytest.MonkeyPatch) -> None:
+ calls: list[Fraction] = []
+ original = collector_module._probability_text
+
+ def _counted_probability_text(probability: Fraction) -> str:
+ calls.append(probability)
+ return original(probability)
+
+ monkeypatch.setattr(collector_module, "_probability_text", _counted_probability_text)
+ with_skill_cases = {f"case-{index}": {"passed": True} for index in range(8)}
+ without_skill_cases = {case_id: {"passed": False} for case_id in with_skill_cases}
+
+ _paired_pass_comparison(
+ {"total_cases": 8, "cases": with_skill_cases},
+ {"total_cases": 8, "cases": without_skill_cases},
+ )
+
+ assert len(calls) == 1
+
+
+ ("discordant", "exact_is_serialized"),
+ [(14_285, True), (14_286, False), (15_000, False)],
+)
+def test_large_exact_pairing_respects_integer_string_safety_limit(
+ discordant: int,
+ exact_is_serialized: bool,
+) -> None:
+ with_skill_cases = {f"case-{index}": {"passed": True} for index in range(discordant)}
+ without_skill_cases = {case_id: {"passed": False} for case_id in with_skill_cases}
+
+ paired = _paired_pass_comparison(
+ {"total_cases": discordant, "cases": with_skill_cases},
+ {"total_cases": discordant, "cases": without_skill_cases},
+ )
+ exact = paired["mcnemar_exact"]
+
+ assert exact["p_value_text"] != "0"
+ assert exact["minimum_attainable_p_value_text"] != "0"
+ assert exact["p_value_exact_omitted"] is not exact_is_serialized
+ assert exact["minimum_attainable_p_value_exact_omitted"] is not exact_is_serialized
+ if exact_is_serialized:
+ assert isinstance(exact["p_value_exact"], str)
+ assert isinstance(exact["minimum_attainable_p_value_exact"], str)
+ else:
+ assert exact["p_value_exact"] is None
+ assert exact["p_value_exact_omitted_reason"] == "decimal_digit_limit"
+ assert exact["minimum_attainable_p_value_exact"] is None
+ assert exact["minimum_attainable_p_value_exact_omitted_reason"] == "decimal_digit_limit"
+
+
+def test_large_exact_pairing_respects_active_integer_string_limit_in_subprocess() -> None:
+ code = """
+from skillevaluator.tier3.harbor.collector import _paired_pass_comparison
+
+pair_count = 2_200
+with_skill_cases = {f"case-{index}": {"passed": True} for index in range(pair_count)}
+without_skill_cases = {case_id: {"passed": False} for case_id in with_skill_cases}
+paired = _paired_pass_comparison(
+ {"total_cases": pair_count, "cases": with_skill_cases},
+ {"total_cases": pair_count, "cases": without_skill_cases},
+)
+exact = paired["mcnemar_exact"]
+assert exact["p_value_exact"] is None
+assert exact["p_value_exact_omitted"] is True
+assert exact["p_value_exact_omitted_reason"] == "decimal_digit_limit"
+assert exact["minimum_attainable_p_value_exact"] is None
+assert exact["minimum_attainable_p_value_exact_omitted"] is True
+assert exact["minimum_attainable_p_value_exact_omitted_reason"] == "decimal_digit_limit"
+"""
+ environment = {**os.environ, "PYTHONINTMAXSTRDIGITS": "640"}
+
+ subprocess.run(
+ [sys.executable, "-c", code],
+ check=True,
+ capture_output=True,
+ text=True,
+ env=environment,
+ )
+
+
+def test_paired_pass_comparison_does_not_issue_exact_test_for_partial_pairing() -> None:
+ paired = _paired_pass_comparison(
+ {"total_cases": 2, "cases": {"shared": {"passed": True}, "with-only": {"passed": True}}},
+ {"total_cases": 2, "cases": {"shared": {"passed": False}, "without-only": {"passed": False}}},
+ )
+
+ assert paired["pairing_status"] == "partial"
+ assert paired["paired_cases"] == 1
+ assert paired["with_skill_unpaired_case_count"] == 1
+ assert paired["without_skill_unpaired_case_count"] == 1
+ assert paired["with_skill_unpaired_case_ids"] == ["with-only"]
+ assert paired["without_skill_unpaired_case_ids"] == ["without-only"]
+ assert paired["with_skill_unpaired_case_ids_truncated"] is False
+ assert paired["without_skill_unpaired_case_ids_truncated"] is False
+ assert "mcnemar_exact" not in paired
+
+
+def test_paired_pass_comparison_bounds_unpaired_case_id_diagnostics() -> None:
+ case_count = 2_400
+ with_cases = {f"with-{index:04d}": {"passed": True} for index in range(case_count)}
+ without_cases = {f"without-{index:04d}": {"passed": False} for index in range(case_count)}
+
+ paired = _paired_pass_comparison(
+ {"total_cases": case_count, "cases": with_cases},
+ {"total_cases": case_count, "cases": without_cases},
+ )
+
+ assert paired["pairing_status"] == "unavailable"
+ assert paired["with_skill_unpaired_case_count"] == case_count
+ assert paired["without_skill_unpaired_case_count"] == case_count
+ assert len(paired["with_skill_unpaired_case_ids"]) == 64
+ assert len(paired["without_skill_unpaired_case_ids"]) == 64
+ assert paired["with_skill_unpaired_case_ids_truncated"] is True
+ assert paired["without_skill_unpaired_case_ids_truncated"] is True
+
+
def test_condition_marks_incomplete_reward_as_unscored() -> None:
reward = {
"entry_id": "case-001",