Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): surface declarative-check coverage in all reports (#586)

Christopher Kevinf4eefdce0d83
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3917cd97..817f7bbf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -426,6 +426,9 @@ All notable changes to SkillEvaluator are documented in this file.
JSON report lists the files in `semgrep_unparsed_files` (up to 1,000, with
`semgrep_unparsed_files_omitted` counting the rest); other Semgrep errors are
reported as before.
+- Tier 3 terminal, JSON, HTML, and benchmark reports now retain declarative-check
+ coverage per agent and arm, including passed/failed counts, judgments,
+ not-checked reasons, and the behavior-score denominators.
- The supporting Tier 3 Pass@k table displays an unavailable or skipped
baseline rate as `N/A`, preserving an observed baseline rate of `0%`.
- Tier 3 budget summaries disclose measured usage above configured limits,
diff --git a/docs/reports.mdx b/docs/reports.mdx
index a93ef416..9d885702 100644
--- a/docs/reports.mdx
+++ b/docs/reports.mdx
@@ -125,6 +125,39 @@ scored totals. Preview limits and physical retry rows do not replace recorded
attempt counts. These are display projections; canonical JSON and evaluation
payloads are unchanged.
+### Tier 3 declarative-check coverage
+
+Runs with declarative checks include a **Declarative-check coverage** section in
+terminal summaries, HTML reports, and both benchmark Markdown cards. Run
+`result.json` and exported report JSON retain the same structured evidence under
+each agent's `declarative_checks`, separated into `with_skill` and
+`without_skill` arms. Reports without declarative checks add no section.
+
+Each arm shows checked totals (passed and failed), judged requirements, and
+not-checked checks. Evidence entries identify the case, trial, check type,
+expected target, verdict, and reason, including passing checks and checks that
+were not applicable without the skill. Not checked is not a pass or a failure.
+
+The behavior-check denominator for each reward is **checked + judged**;
+not-checked checks are excluded. Reports show both per-reward denominators and
+their sum across an arm. That sum is coverage, not a pooled behavior score:
+existing trial averaging and scoring remain unchanged. A judge error still
+produces no behavior score, even when its recorded denominator is nonzero.
+Missing judgment counts are unavailable rather than assumed to be zero.
+
+Coverage uses the reward artifacts already loaded by reporting, not a new
+evaluation or a fresh judgment. Failed trials without reward evidence do not
+gain invented checks. For retained runs, only available reward evidence can be
+reconstructed; existing report truncation notices still apply.
+
+A very large run can make this evidence exceed the result and report size
+limits. Then `result.json` and the report keep at most 200 check rows per arm,
+failed and not-checked rows first, and leave out the rest. Totals, judged
+counts, and per-reward denominators stay exact. Each trimmed arm records
+`omitted_checks`, the summary says how many rows were left out, and the report
+notes them as `declarative_check_rows` under `report_truncation.omitted`. The
+full rows remain in each trial's `reward.json`.
+
### Tier 2 comparison scopes and evidence
The HTML Tier 2 view separates **Within this skill** (redundant content among
diff --git a/src/skillevaluator/evaluation/tier3_report.py b/src/skillevaluator/evaluation/tier3_report.py
index df0a0821..c4e49c64 100644
--- a/src/skillevaluator/evaluation/tier3_report.py
+++ b/src/skillevaluator/evaluation/tier3_report.py
@@ -51,6 +51,11 @@ from skillevaluator.evaluation.tier3_findings import (
from skillevaluator.evidence import evidence_ref_identity
from skillevaluator.models.result import Finding, Severity, ValidationResult
from skillevaluator.source_identity import resolve_evaluated_source
+from skillevaluator.tier3.declarative_coverage import (
+ COMPACT_CHECK_ROWS_PER_ARM,
+ agent_declarative_checks,
+ compact_declarative_checks,
+)
from skillevaluator.tier3.evidence_volume import evidence_volume_for_conditions
logger = logging.getLogger(__name__)
@@ -200,6 +205,7 @@ class _ReportBudget:
"raw_metrics_per_reward": _MAX_RAW_METRICS_PER_REWARD,
"custom_metric_name_visits_per_reward": _MAX_CUSTOM_METRIC_NAME_VISITS_PER_REWARD,
"unpaired_case_id_samples": _MAX_UNPAIRED_CASE_IDS_IN_REPORT,
+ "declarative_check_rows_per_arm": COMPACT_CHECK_ROWS_PER_ARM,
},
"omitted": dict(sorted(self.omitted.items())),
}
@@ -859,6 +865,12 @@ def build_agent_eval_payload(
baseline_only=baseline_only,
)
+ coverage = info.get("declarative_checks") or agent_declarative_checks(
+ info.get("rewards") or [], info.get("rewards_baseline") or []
+ )
+ if coverage:
+ agent_payloads[name]["declarative_checks"] = coverage
+
if not agent_payloads:
return None
@@ -1551,6 +1563,25 @@ def _prune_pass_at_k_pairing_diagnostics(payload: dict[str, Any], report_budget:
report_budget.omit("unpaired_case_ids", omitted)
+def _prune_declarative_check_rows(payload: dict[str, Any], report_budget: _ReportBudget) -> None:
+ """Cap per-check coverage rows when the payload is over its limits; counts stay exact.
+
+ Coverage grows with every check of every reward, so it goes before any scored detail.
+ """
+ agents = [
+ agent
+ for agent in (payload.get("agents") or {}).values()
+ if isinstance(agent, dict) and isinstance(agent.get("declarative_checks"), dict)
+ ]
+ if not agents or _within_report_limits(payload):
+ return
+ omitted = 0
+ for agent in agents:
+ agent["declarative_checks"], dropped = compact_declarative_checks(agent["declarative_checks"])
+ omitted += dropped
+ report_budget.omit("declarative_check_rows", omitted)
+
+
def _enforce_report_payload_budget(
payload: dict[str, Any], report_budget: _ReportBudget, *, enforce_size: bool = True
) -> None:
@@ -1572,7 +1603,11 @@ def _enforce_report_payload_budget(
# best-agent and top-level projections.
_prune_pass_at_k_pairing_diagnostics(payload, report_budget)
refresh_signal()
- if not enforce_size or _serialized_payload_size(payload) <= _MAX_EMBEDDED_REPORT_BYTES:
+ if not enforce_size:
+ return
+ _prune_declarative_check_rows(payload, report_budget)
+ refresh_signal()
+ if _serialized_payload_size(payload) <= _MAX_EMBEDDED_REPORT_BYTES:
return
# Built failure reasons and excerpts only explain scores the payload already
@@ -1752,12 +1787,10 @@ def _report_omissions(payload: dict[str, Any]) -> dict[str, Any] | None:
return omitted if isinstance(omitted, dict) else None
-def _token_usage_fits(payload: dict[str, Any]) -> bool:
+def _within_report_limits(payload: dict[str, Any]) -> bool:
+ """Whether the payload fits the embedded size budget and the reporters' node, depth and text limits."""
from skillevaluator.reporting.base import agent_eval_report_serialization_issue
- # The minimal last-resort payload carries no details.
- if "payload_sections" in (_report_omissions(payload) or {}):
- return False
try:
size = _serialized_payload_size(payload)
except (TypeError, ValueError):
@@ -1765,6 +1798,13 @@ def _token_usage_fits(payload: dict[str, Any]) -> bool:
return size <= _MAX_EMBEDDED_REPORT_BYTES and agent_eval_report_serialization_issue(payload) is None
+def _token_usage_fits(payload: dict[str, Any]) -> bool:
+ # The minimal last-resort payload carries no details.
+ if "payload_sections" in (_report_omissions(payload) or {}):
+ return False
+ return _within_report_limits(payload)
+
+
def _record_token_usage_omission(payload: dict[str, Any], section: str, count: int) -> bool:
"""Count a token usage drop in ``report_truncation``; ``False`` when its record cannot take it."""
if "report_truncation" not in payload:
diff --git a/src/skillevaluator/reporting/benchmark.py b/src/skillevaluator/reporting/benchmark.py
index 4086bd9c..c20ebb1a 100644
--- a/src/skillevaluator/reporting/benchmark.py
+++ b/src/skillevaluator/reporting/benchmark.py
@@ -53,6 +53,13 @@ from skillevaluator.reporting.base import (
tier1_unscanned_publication_paths,
unrecognized_tier2_results,
)
+from skillevaluator.reporting.declarative_coverage import (
+ DENOMINATOR_NOTE,
+ attempt_denominators,
+ check_status,
+ coverage_summary,
+ declarative_coverage_groups,
+)
from skillevaluator.reporting.run_status import resolve_run_status
from skillevaluator.reporting.summary_copy import SKILL_LIFT_DEFINITION, pass_at_k_summary
from skillevaluator.reporting.tier3_coverage import IncompleteArm, incomplete_arms
@@ -331,6 +338,34 @@ class BenchmarkReporter(ReporterBase):
self._render_results_at_a_glance(lines, ae, private_labels)
lines.extend(_agent_coverage_lines(ae, private_labels))
lines.extend(_token_usage_lines(ae, private_labels))
+ groups = declarative_coverage_groups((ae or {}).get("agents"))
+ if groups:
+ lines.extend(["## Declarative-check coverage", "", DENOMINATOR_NOTE, ""])
+ for group in groups:
+ lines.extend(
+ [
+ f"### {_md_cell(group['agent'], private_labels)} / {group['arm']}",
+ "",
+ coverage_summary(group),
+ "",
+ "Per-reward denominators: " + _md_cell(attempt_denominators(group), private_labels),
+ "",
+ "| Case / trial | Check | Expected | Status | Reason |",
+ "| --- | --- | --- | --- | --- |",
+ ]
+ )
+ for item in group["results"]:
+ values = (
+ item["case_id"],
+ item["trial_id"],
+ item["type"],
+ item["expected"],
+ check_status(item),
+ item["reason"],
+ )
+ case, trial, *evidence = (_md_cell(value, private_labels) for value in values)
+ lines.append("| " + " | ".join([f"{case} / {trial}", *evidence]) + " |")
+ lines.append("")
self._render_tier_status(
lines,
results,
diff --git a/src/skillevaluator/reporting/cli.py b/src/skillevaluator/reporting/cli.py
index 9e11b7c2..8622da76 100644
--- a/src/skillevaluator/reporting/cli.py
+++ b/src/skillevaluator/reporting/cli.py
@@ -47,6 +47,7 @@ from skillevaluator.reporting.base import (
source_validation_errors,
source_validation_preview,
)
+from skillevaluator.reporting.declarative_coverage import render_declarative_coverage
from skillevaluator.reporting.finding_verification import finding_verification
from skillevaluator.reporting.harbor_viewer import (
harbor_evidence_link_text,
@@ -452,6 +453,7 @@ class CLIReporter(ReporterBase):
@staticmethod
def _print_agent_eval_tables(agent_eval: dict, console: Console) -> None:
"""Print Tier 3 agent evaluation results."""
+ render_declarative_coverage(console, agent_eval.get("agents"))
if agent_eval.get("baseline"):
console.print(escape_markup(baseline_description(agent_eval["baseline"])))
verdict = agent_eval.get("verdict", "unknown")
diff --git a/src/skillevaluator/reporting/dashboard_data.py b/src/skillevaluator/reporting/dashboard_data.py
index 3f18cb8d..cc2d062d 100644
--- a/src/skillevaluator/reporting/dashboard_data.py
+++ b/src/skillevaluator/reporting/dashboard_data.py
@@ -136,6 +136,7 @@ def _trial_coverage_truncated(marker: dict[str, Any]) -> bool:
"raw_reward_fields",
"raw_metric_values",
"unpaired_case_ids",
+ "declarative_check_rows",
"comparison_payloads",
"evidence_entries",
"evaluator_cards",
diff --git a/src/skillevaluator/reporting/declarative_coverage.py b/src/skillevaluator/reporting/declarative_coverage.py
new file mode 100644
index 00000000..8a08d6b6
--- /dev/null
+++ b/src/skillevaluator/reporting/declarative_coverage.py
@@ -0,0 +1,91 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Shared display projections for recorded declarative-check coverage."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from typing import TYPE_CHECKING, Any
+
+from rich.table import Table
+from rich.text import Text
+
+if TYPE_CHECKING:
+ from rich.console import Console
+
+DENOMINATOR_NOTE = (
+ "Behavior denominator = checked + judged per reward; not checked is excluded. "
+ "Totals sum these denominators, not a pooled behavior score. A judge error still yields no score."
+)
+
+
+def declarative_coverage_groups(agents: object) -> list[dict[str, Any]]:
+ groups = []
+ if not isinstance(agents, Mapping):
+ return groups
+ for name, agent in agents.items():
+ coverage = agent.get("declarative_checks") if isinstance(agent, Mapping) else None
+ if not isinstance(coverage, Mapping):
+ continue
+ for arm in ("with_skill", "without_skill"):
+ data = coverage.get(arm)
+ if not isinstance(data, dict) or not data.get("results"):
+ continue
+ groups.append({**data, "agent": str(name), "arm": arm.replace("_", " ")})
+ return groups
+
+
+def coverage_summary(group: dict[str, Any]) -> str:
+ def count(key: str) -> str:
+ value = group.get(key)
+ return str(value) if value is not None else "unavailable"
+
+ omitted = group.get("omitted_checks")
+ shown = f"; {omitted} more check rows omitted to fit report limits" if type(omitted) is int and omitted > 0 else ""
+ return (
+ f"Checked {count('checked_count')} ({count('passed_count')} passed / {count('failed_count')} failed); "
+ f"judged {count('judged_count')}; not checked {count('not_checked_count')}; "
+ f"behavior denominator {count('score_denominator')}{shown}"
+ )
+
+
+def check_status(item: dict[str, Any]) -> str:
+ return "passed" if item.get("passed") is True else "failed" if item.get("passed") is False else "not checked"
+
+
+def attempt_denominators(group: dict[str, Any]) -> str:
+ return "; ".join(
+ f"{item['case_id']} [{item['trial_id']}]: "
+ f"{item['score_denominator'] if item['score_denominator'] is not None else 'unavailable'} "
+ f"({item['score_status']})"
+ for item in group.get("attempts", [])
+ )
+
+
+def render_declarative_coverage(console: Console, agents: object, *, text: Callable[[str], str] = str) -> None:
+ groups = declarative_coverage_groups(agents)
+ if not groups:
+ return
+ console.print(Text("\nDeclarative-check coverage", style="bold"))
+ console.print(Text(DENOMINATOR_NOTE, style="dim"))
+ for group in groups:
+ console.print(Text(text(f"{group['agent']} / {group['arm']}: {coverage_summary(group)}")))
+ console.print(Text(text("Per-reward denominators: " + attempt_denominators(group)), style="dim"))
+ table = Table(expand=True)
+ for label in ("Case / trial", "Check", "Expected", "Status", "Reason"):
+ table.add_column(label, overflow="fold")
+ for item in group["results"]:
+ table.add_row(
+ *(
+ Text(text(str(value)))
+ for value in (
+ f"{item['case_id']} / {item['trial_id']}",
+ item["type"],
+ item["expected"],
+ check_status(item),
+ item["reason"],
+ )
+ )
+ )
+ console.print(table)
diff --git a/src/skillevaluator/reporting/html.py b/src/skillevaluator/reporting/html.py
index bd65021a..614171c7 100644
--- a/src/skillevaluator/reporting/html.py
+++ b/src/skillevaluator/reporting/html.py
@@ -80,6 +80,13 @@ from skillevaluator.reporting.base import (
from skillevaluator.reporting.base import (
is_tier2_validator_name as _is_tier2_validator_name,
)
+from skillevaluator.reporting.declarative_coverage import (
+ DENOMINATOR_NOTE,
+ attempt_denominators,
+ check_status,
+ coverage_summary,
+ declarative_coverage_groups,
+)
from skillevaluator.reporting.finding_summary import (
describe_issue_counts,
finding_counts,
@@ -3621,6 +3628,13 @@ class HTMLReporter(ReporterBase):
tier3_truncation=tier3_truncation,
tier3_dataset_sufficiency=tier3_dataset_sufficiency,
tier3_preview_notice=tier3_preview_notice,
+ declarative_coverage_groups=declarative_coverage_groups(
+ tier3_data.get("agents") if isinstance(tier3_data, dict) else {}
+ ),
+ declarative_coverage_summary=coverage_summary,
+ declarative_check_status=check_status,
+ declarative_attempt_denominators=attempt_denominators,
+ declarative_denominator_note=DENOMINATOR_NOTE,
# Measured, not scored. From the full display payload, with its own
# per-agent cap: the preview walk can blank strings of late keys.
tier3_token_usage=token_usage_table(
diff --git a/src/skillevaluator/reporting/templates/report.html.j2 b/src/skillevaluator/reporting/templates/report.html.j2
index fdf478cf..a53d36b5 100644
--- a/src/skillevaluator/reporting/templates/report.html.j2
+++ b/src/skillevaluator/reporting/templates/report.html.j2
@@ -3225,6 +3225,22 @@
</details>
{% endif %}
+ {% if declarative_coverage_groups %}
+ <h3>Declarative-check coverage</h3>
+ <p class="t3-section-hint">{{ declarative_denominator_note }}</p>
+ {% for group in declarative_coverage_groups %}
+ <h4>{{ group.agent }} / {{ group.arm }}</h4>
+ <p>{{ declarative_coverage_summary(group) }}</p>
+ <p class="t3-section-hint">Per-reward denominators: {{ declarative_attempt_denominators(group) }}</p>
+ <div class="table-scroll"><table class="summary-table">
+ <thead><tr><th>Case / trial</th><th>Check</th><th>Expected</th><th>Status</th><th>Reason</th></tr></thead>
+ <tbody>{% for item in group.results %}
+ <tr><td>{{ item.case_id }} / {{ item.trial_id }}</td><td>{{ item.type }}</td><td>{{ item.expected }}</td><td>{{ declarative_check_status(item) }}</td><td>{{ item.reason }}</td></tr>
+ {% endfor %}</tbody>
+ </table></div>
+ {% endfor %}
+ {% endif %}
+
{% if t3_condition_trial_rows %}
{% set t3_trial_agent_groups = (tier3.trials or []) | groupby('agent') %}
diff --git a/src/skillevaluator/tier3/declarative_coverage.py b/src/skillevaluator/tier3/declarative_coverage.py
new file mode 100644
index 00000000..0fe92f73
--- /dev/null
+++ b/src/skillevaluator/tier3/declarative_coverage.py
@@ -0,0 +1,100 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Recorded declarative-check coverage, independent of task success and skill lift."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from skillevaluator.evaluation.tier3_findings import sanitize_grader_text
+
+# Per-check rows kept for each arm once a result or report is over its size limits.
+COMPACT_CHECK_ROWS_PER_ARM = 200
+
+
+def aggregate_declarative_checks(rewards: list[dict[str, Any]]) -> dict[str, Any]:
+ """Keep every check and each reward's denominator; never pool behavior scores."""
+ totals = dict.fromkeys(
+ ("checked_count", "passed_count", "failed_count", "judged_count", "not_checked_count", "score_denominator"), 0
+ )
+ results: list[dict[str, Any]] = []
+ attempts: list[dict[str, Any]] = []
+ for reward in rewards:
+ details = reward.get("details")
+ behavior = details.get("behavior_check") if isinstance(details, dict) else None
+ coverage = behavior.get("declarative_checks") if isinstance(behavior, dict) else None
+ if not isinstance(coverage, dict) or not isinstance(coverage.get("results"), list):
+ continue
+ checks = [item for item in coverage["results"] if isinstance(item, dict)]
+ identity = {
+ "case_id": sanitize_grader_text(str(reward.get("entry_id") or "unknown")),
+ "trial_id": sanitize_grader_text(str(reward.get("_trial_name") or reward.get("trial_id") or "")),
+ }
+ passed = sum(item.get("passed") is True for item in checks)
+ failed = sum(item.get("passed") is False for item in checks)
+ judged = coverage.get("judged_count")
+ judged = judged if type(judged) is int and judged >= 0 else None
+ denominator = passed + failed + judged if judged is not None else None
+ counts = {
+ "checked_count": passed + failed,
+ "passed_count": passed,
+ "failed_count": failed,
+ "judged_count": judged,
+ "not_checked_count": len(checks) - passed - failed,
+ "score_denominator": denominator,
+ }
+ for key, value in counts.items():
+ totals[key] = totals[key] + value if totals[key] is not None and value is not None else None
+ attempts.append(
+ {**identity, **counts, "score_status": sanitize_grader_text(behavior.get("status") or "unknown")}
+ )
+ for item in checks:
+ verdict = item.get("passed")
+ results.append(
+ {
+ **identity,
+ "type": sanitize_grader_text(str(item.get("type") or "unknown")),
+ "expected": sanitize_grader_text(str(item.get("expected") or "")),
+ "passed": verdict if type(verdict) is bool else None,
+ "reason": sanitize_grader_text(str(item.get("reason") or "")),
+ }
+ )
+ return {**totals, "attempts": attempts, "results": results} if results else {}
+
+
+def agent_declarative_checks(
+ with_rewards: list[dict[str, Any]], without_rewards: list[dict[str, Any]]
+) -> dict[str, Any]:
+ """Omit absent arms and omit the entire section for non-declarative runs."""
+ return {
+ arm: coverage
+ for arm, rewards in (("with_skill", with_rewards), ("without_skill", without_rewards))
+ if (coverage := aggregate_declarative_checks(rewards))
+ }
+
+
+def compact_declarative_checks(
+ coverage: dict[str, Any], *, max_rows: int = COMPACT_CHECK_ROWS_PER_ARM
+) -> tuple[dict[str, Any], int]:
+ """Cap each arm's per-check rows, keeping exact counts and recording what was left out.
+
+ Failed and not-checked rows are kept ahead of passed ones. The totals and the
+ per-reward ``attempts`` do not change; ``omitted_checks`` counts the dropped rows.
+ The input is never modified. Returns the compact copy and the rows dropped.
+ """
+ compact: dict[str, Any] = {}
+ dropped_total = 0
+ for arm, data in coverage.items():
+ rows = data.get("results") if isinstance(data, dict) else None
+ if not isinstance(rows, list) or len(rows) <= max_rows:
+ compact[arm] = data
+ continue
+ order = sorted(range(len(rows)), key=lambda i: (isinstance(rows[i], dict) and rows[i].get("passed") is True, i))
+ kept = sorted(order[:max_rows])
+ dropped = len(rows) - len(kept)
+ earlier = data.get("omitted_checks")
+ earlier = earlier if type(earlier) is int and earlier > 0 else 0
+ compact[arm] = {**data, "results": [rows[i] for i in kept], "omitted_checks": earlier + dropped}
+ dropped_total += dropped
+ return compact, dropped_total
diff --git a/src/skillevaluator/tier3/harbor/collector.py b/src/skillevaluator/tier3/harbor/collector.py
index 9dd76f88..5daee0be 100644
--- a/src/skillevaluator/tier3/harbor/collector.py
+++ b/src/skillevaluator/tier3/harbor/collector.py
@@ -30,6 +30,7 @@ from typing import Any
from skillevaluator.tier3.attempt_verdict import attempt_verdict, scored_goal_accuracy
from skillevaluator.tier3.baseline_store import BASELINE_ANNOTATION_KEYS, load_baseline_rows, measured_condition_usage
+from skillevaluator.tier3.declarative_coverage import agent_declarative_checks
from skillevaluator.tier3.eval_core.atif_helpers import extract_tool_calls_as_dicts, get_skill_tool_calls
from skillevaluator.tier3.eval_core.checks import check_negative_case
from skillevaluator.tier3.eval_core.codex_tool_call_normalizer import atif_content_text
@@ -8346,6 +8347,9 @@ def collect_harbor_results(
},
"output_dir": str(agent_dir.resolve()),
}
+ coverage = agent_declarative_checks(with_collected_rewards, without_collected_rewards)
+ if coverage:
+ all_results["agents"][agent]["declarative_checks"] = coverage
if (
tolerance_on
and without_job_dir is not None
diff --git a/src/skillevaluator/tier3/harbor/report_data.py b/src/skillevaluator/tier3/harbor/report_data.py
index 06e5b562..848942bf 100644
--- a/src/skillevaluator/tier3/harbor/report_data.py
+++ b/src/skillevaluator/tier3/harbor/report_data.py
@@ -21,6 +21,7 @@ from itertools import islice
from pathlib import Path
from typing import Any
+from skillevaluator.tier3.declarative_coverage import agent_declarative_checks
from skillevaluator.tier3.eval_core.checks import resolve_should_trigger
from skillevaluator.tier3.harbor import baseline_timeouts
from skillevaluator.tier3.harbor.metrics import (
@@ -1153,6 +1154,10 @@ def load_agent_data(
}
)
+ coverage = agent_declarative_checks(agent_info.get("rewards") or [], agent_info.get("rewards_baseline") or [])
+ if coverage:
+ agent_info["declarative_checks"] = coverage
+
condition_quality_fields = {
"with_skill": (
"with_skill",
diff --git a/src/skillevaluator/tier3/harbor/runner.py b/src/skillevaluator/tier3/harbor/runner.py
index 1aeecf74..d2f36891 100644
--- a/src/skillevaluator/tier3/harbor/runner.py
+++ b/src/skillevaluator/tier3/harbor/runner.py
@@ -76,6 +76,7 @@ from skillevaluator.tier3.baseline_store import (
select_baseline,
)
from skillevaluator.tier3.case_ids import validate_case_ids
+from skillevaluator.tier3.declarative_coverage import compact_declarative_checks
from skillevaluator.tier3.eval_core.llm_judge import JUDGE_REASONING_EFFORT
from skillevaluator.tier3.evals_config import (
_GKE_INFRASTRUCTURE_KWARGS,
@@ -586,6 +587,18 @@ def _compact_agent_result(
compact["execution_error_details_truncated"] = len(compact["execution_errors"]) < exact_total
omitted.append("execution_errors")
+ raw_coverage = compact.get("declarative_checks")
+ if isinstance(raw_coverage, dict):
+ # Cap the per-check rows; every count and per-reward denominator stays exact.
+ compact_coverage, dropped_checks = compact_declarative_checks(raw_coverage)
+ if dropped_checks:
+ compact["declarative_checks"] = compact_coverage
+ omitted.extend(
+ f"declarative_checks.{arm}.results"
+ for arm, data in compact_coverage.items()
+ if data is not raw_coverage[arm]
+ )
+
if omitted:
compact["detail_projection"] = {
"artifacts": sorted(references),
diff --git a/src/skillevaluator/tier3/result_display.py b/src/skillevaluator/tier3/result_display.py
index 29bb898a..ac5ea79c 100644
--- a/src/skillevaluator/tier3/result_display.py
+++ b/src/skillevaluator/tier3/result_display.py
@@ -20,6 +20,7 @@ from rich.panel import Panel
from rich.table import Table
from rich.text import Text
+from skillevaluator.reporting.declarative_coverage import render_declarative_coverage
from skillevaluator.reporting.harbor_viewer import harbor_job_links, normalize_harbor_viewer_for_display
from skillevaluator.reporting.summary_copy import SKILL_LIFT_DEFINITION
from skillevaluator.reporting.terminal_styles import AMBER, FAINT, GREEN, RED, TEXT, lift_style
@@ -1281,6 +1282,8 @@ def render_evaluation_result(
# A run whose arms all failed has no scores to show, but its attempts still used tokens.
_render_token_usage(console, result, safe)
+ render_declarative_coverage(console, agents, text=safe)
+
# A recovered timeout is neither a failure nor a score; say so instead of staying silent.
for line in _recovered_timeout_lines(agents) if isinstance(agents, Mapping) else []:
console.print(Text(safe(line), style="yellow"))
diff --git a/tests/reporting/test_dashboard_data.py b/tests/reporting/test_dashboard_data.py
index b93cc687..00d6867f 100644
--- a/tests/reporting/test_dashboard_data.py
+++ b/tests/reporting/test_dashboard_data.py
@@ -539,7 +539,9 @@ def test_explicit_adverse_row_has_no_invented_baseline_delta() -> None:
assert len(comparison_rows(data.rows)) == 1
[email protected]("omitted", ["evidence_entries", "raw_trial_rewards", "non_best_agent_details"])
+ "omitted", ["evidence_entries", "raw_trial_rewards", "non_best_agent_details", "declarative_check_rows"]
+)
def test_diagnostic_truncation_preserves_complete_trial_comparisons(omitted: str) -> None:
report = _report()
report["report_truncation"] = {"truncated": True, "omitted": {omitted: 50}}
diff --git a/tests/reporting/test_declarative_coverage.py b/tests/reporting/test_declarative_coverage.py
new file mode 100644
index 00000000..bb3c8d1a
--- /dev/null
+++ b/tests/reporting/test_declarative_coverage.py
@@ -0,0 +1,478 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Declarative coverage survives aggregation and every supported report surface."""
+
+from __future__ import annotations
+
+import copy
+import io
+import json
+from html.parser import HTMLParser
+
+import pytest
+from rich.console import Console
+
+from skillevaluator.evaluation.tier3_report import agent_eval_result_from_directory, build_agent_eval_payload
+from skillevaluator.models import ValidationResult
+from skillevaluator.reporting import BenchmarkReporter, CLIReporter, HTMLReporter, JSONReporter
+from skillevaluator.reporting.base import agent_eval_report_serialization_issue
+from skillevaluator.reporting.declarative_coverage import coverage_summary
+from skillevaluator.tier3.declarative_coverage import (
+ COMPACT_CHECK_ROWS_PER_ARM,
+ agent_declarative_checks,
+ aggregate_declarative_checks,
+ compact_declarative_checks,
+)
+from skillevaluator.tier3.harbor.collector import collect_harbor_results
+from skillevaluator.tier3.harbor.metrics import DEFAULT_METRICS
+from skillevaluator.tier3.result_display import render_evaluation_result
+
+
+class _VisibleHTMLText(HTMLParser):
+ def __init__(self):
+ super().__init__()
+ self.hidden = False
+ self.parts = []
+
+ def handle_starttag(self, tag, _attrs):
+ if tag in {"script", "style"}:
+ self.hidden = True
+
+ def handle_endtag(self, tag):
+ if tag in {"script", "style"}:
+ self.hidden = False
+
+ def handle_data(self, data):
+ if not self.hidden:
+ self.parts.append(data)
+
+
[email protected]("tag", ["script", "SCRIPT", "script ", "style"])
+def test_visible_html_text_excludes_embedded_data(tag):
+ parser = _VisibleHTMLText()
+ parser.feed(f"<p>Visible evidence</p><{tag}>Embedded data</{tag}><p>More evidence</p>")
+ parser.close()
+ assert " ".join(parser.parts) == "Visible evidence More evidence"
+
+
+def _reward(*, baseline=False):
+ checks = [
+ {"type": "script_invoked", "expected": "scripts/build.py", "passed": True, "reason": "Invoked build script"},
+ {
+ "type": "command_succeeded",
+ "expected": "scripts/build.py",
+ "passed": False,
+ "reason": "Invoked but exited 2",
+ },
+ {"type": "file_exists", "expected": "out/report.md", "passed": None, "reason": "File evidence unavailable"},
+ ]
+ if baseline:
+ for check in checks:
+ check.update(passed=None, reason="Not applicable without the skill")
+ return {
+ **dict.fromkeys(DEFAULT_METRICS, 0.8),
+ "overall": 0.8,
+ "entry_id": "build-case",
+ "_trial_name": "build-case__baseline" if baseline else "build-case__attempt",
+ "details": {
+ "behavior_check": {
+ "score": 0.5,
+ "status": "accepted",
+ "declarative_checks": {
+ "results": checks,
+ "judged_count": 2,
+ "checked_count": 0 if baseline else 2,
+ "not_checked_count": 3 if baseline else 1,
+ },
+ }
+ },
+ }
+
+
+def _info(*, checks=True):
+ rewards = [_reward()]
+ baseline = [_reward(baseline=True)]
+ if not checks:
+ for reward in rewards + baseline:
+ reward["details"] = {}
+ return {
+ "with_skill": dict.fromkeys(DEFAULT_METRICS, 0.8),
+ "without_skill": dict.fromkeys(DEFAULT_METRICS, 0.8),
+ "execution_status": "succeeded",
+ "expected_attempts": 2,
+ "scored_attempts": 2,
+ "conditions": {
+ arm: {"execution_status": "succeeded", "expected_attempts": 1, "scored_attempts": 1}
+ for arm in ("with_skill", "without_skill")
+ },
+ "rewards": rewards,
+ "rewards_baseline": baseline,
+ }
+
+
+def _payload(*, checks=True):
+ info = _info(checks=checks)
+ return build_agent_eval_payload("demo", {"agent-a": info, "agent-b": copy.deepcopy(info)}, use_llm_judge=False)
+
+
+def _validation(payload):
+ return ValidationResult(validator_name="AGENT_EVAL", metadata={"agent_eval": payload})
+
+
+def test_coverage_counts_exclude_not_checked_and_do_not_mutate_rewards():
+ rewards = [_reward(), _reward(baseline=True)]
+ original = copy.deepcopy(rewards)
+ coverage = aggregate_declarative_checks(rewards)
+ assert {
+ key: coverage[key]
+ for key in (
+ "checked_count",
+ "passed_count",
+ "failed_count",
+ "judged_count",
+ "not_checked_count",
+ "score_denominator",
+ )
+ } == {
+ "checked_count": 2,
+ "passed_count": 1,
+ "failed_count": 1,
+ "judged_count": 4,
+ "not_checked_count": 4,
+ "score_denominator": 6,
+ }
+ assert [item["score_denominator"] for item in coverage["attempts"]] == [4, 2]
+ assert len(coverage["results"]) == 6
+ assert rewards == original
+
+
+ "rewards",
+ [
+ [],
+ [{"details": {}}],
+ [{"details": {"behavior_check": {"declarative_checks": {"results": [], "judged_count": 2}}}}],
+ ],
+)
+def test_no_declarative_checks_omit_section(rewards):
+ assert aggregate_declarative_checks(rewards) == {}
+ assert agent_declarative_checks(rewards, []) == {}
+
+
+def test_error_and_unknown_judgment_counts_do_not_invent_a_score():
+ reward = _reward()
+ reward["details"]["behavior_check"].update(status="error", score=None)
+ reward["details"]["behavior_check"]["declarative_checks"].pop("judged_count")
+ coverage = aggregate_declarative_checks([reward])
+ assert coverage["judged_count"] is None
+ assert coverage["score_denominator"] is None
+ assert coverage["attempts"][0]["score_status"] == "error"
+ assert coverage["checked_count"] == 2
+
+
+def test_canonical_payload_keeps_both_agents_and_arms():
+ payload = _payload()
+ for agent in payload["agents"].values():
+ coverage = agent["declarative_checks"]
+ assert coverage["with_skill"]["checked_count"] == 2
+ assert coverage["with_skill"]["score_denominator"] == 4
+ assert coverage["without_skill"]["not_checked_count"] == 3
+ assert coverage["without_skill"]["score_denominator"] == 2
+ assert coverage["with_skill"]["results"][1]["reason"] == "Invoked but exited 2"
+
+
+def _terminal(payload, *, verbose):
+ stream = io.StringIO()
+ console = Console(file=stream, width=220, force_terminal=False, color_system=None)
+ render_evaluation_result({"agents": payload["agents"]}, console=console, verbose=verbose)
+ return stream.getvalue()
+
+
[email protected]("verbose", [True, False])
+def test_evaluate_terminal_shows_coverage_even_in_compact_mode(verbose):
+ text = _terminal(_payload(), verbose=verbose)
+ assert "Declarative-check coverage" in text
+ assert "Checked 2 (1 passed / 1 failed); judged 2; not checked 1; behavior denominator 4" in text
+ for value in (
+ "agent-a",
+ "agent-b",
+ "with skill",
+ "without skill",
+ "build-case",
+ "command_succeeded",
+ "scripts/build.py",
+ "Invoked but exited 2",
+ "File evidence unavailable",
+ "Not applicable without the skill",
+ ):
+ assert value in text
+
+
+def test_validate_terminal_shows_coverage():
+ stream = io.StringIO()
+ console = Console(file=stream, width=220, force_terminal=False, color_system=None)
+ CLIReporter._print_agent_eval_tables(_payload(), console)
+ text = stream.getvalue()
+ assert "behavior denominator 4" in text
+ assert "File evidence unavailable" in text
+
+
+ "reporter",
+ [
+ HTMLReporter(report_scope="tier3"),
+ BenchmarkReporter(skill_name="demo", report_scope="tier3"),
+ ],
+)
+def test_html_and_both_benchmark_cards_show_all_checks(reporter):
+ output = reporter.render(_validation(_payload()))
+ visible = output
+ if isinstance(reporter, HTMLReporter):
+ parser = _VisibleHTMLText()
+ parser.feed(output)
+ parser.close()
+ visible = " ".join(parser.parts)
+ if isinstance(reporter, BenchmarkReporter):
+ visible = visible.replace("\\_", "_")
+ for value in (
+ "Declarative-check coverage",
+ "Checked 2 (1 passed / 1 failed)",
+ "judged 2; not checked 1; behavior denominator 4",
+ "agent-a",
+ "agent-b",
+ "build-case",
+ "script_invoked",
+ "command_succeeded",
+ "scripts/build.py",
+ "Invoked but exited 2",
+ "File evidence unavailable",
+ "Not applicable without the skill",
+ ):
+ assert value in visible
+
+
+def test_exported_report_json_keeps_structured_counts_and_evidence():
+ output = json.loads(JSONReporter().render(_validation(_payload())))
+ assert "declarative_checks" in json.dumps(output)
+ encoded = json.dumps(output)
+ for value in (
+ '"score_denominator": 4',
+ '"passed_count": 1',
+ '"failed_count": 1',
+ '"judged_count": 2',
+ '"not_checked_count": 3',
+ '"passed": null',
+ "File evidence unavailable",
+ "Not applicable without the skill",
+ ):
+ assert value in encoded
+
+
+def test_html_escapes_untrusted_check_evidence():
+ payload = _payload()
+ for agent in payload["agents"].values():
+ agent["declarative_checks"]["with_skill"]["results"][0]["reason"] = '<img src=x onerror="alert(1)">'
+ output = HTMLReporter(report_scope="tier3").render(_validation(payload))
+ assert '<img src=x onerror="alert(1)">' not in output
+ assert "&lt;img" in output
+
+
+def test_coverage_redacts_configured_credentials(monkeypatch):
+ credential = "example-private-credential-value"
+ monkeypatch.setenv("OPENAI_API_KEY", credential)
+ reward = _reward()
+ reward["details"]["behavior_check"]["declarative_checks"]["results"][0]["reason"] = f"Error: {credential}"
+ assert credential not in json.dumps(aggregate_declarative_checks([reward]))
+
+
[email protected]("format_name", ["terminal", "cli", "html", "benchmark", "json"])
+def test_non_declarative_reports_add_no_coverage(format_name):
+ payload = _payload(checks=False)
+ assert all("declarative_checks" not in agent for agent in payload["agents"].values())
+ result = _validation(payload)
+ if format_name == "terminal":
+ output = _terminal(payload, verbose=False)
+ else:
+ reporter = {
+ "cli": CLIReporter(),
+ "html": HTMLReporter(report_scope="tier3"),
+ "benchmark": BenchmarkReporter(skill_name="demo", report_scope="tier3"),
+ "json": JSONReporter(),
+ }[format_name]
+ output = reporter.render(result)
+ assert "Declarative-check coverage" not in output
+ assert '"declarative_checks"' not in output
+
+
+def test_collector_result_json_carries_coverage(tmp_path):
+ jobs = tmp_path / "jobs"
+ for suffix, baseline in (("with", False), ("without", True)):
+ job = jobs / f"demo-opencode-{suffix}"
+ verifier = job / "build-case__attempt" / "verifier"
+ verifier.mkdir(parents=True)
+ reward = _reward(baseline=baseline)
+ reward.pop("_trial_name")
+ (verifier / "reward.json").write_text(json.dumps(reward))
+ (job / "build-case__attempt" / "result.json").write_text(json.dumps({"trial_name": "build-case__attempt"}))
+ (job / "result.json").write_text(
+ json.dumps(
+ {
+ "n_total_trials": 1,
+ "stats": {
+ "n_completed_trials": 1,
+ "n_errored_trials": 0,
+ "n_running_trials": 0,
+ "n_pending_trials": 0,
+ "n_cancelled_trials": 0,
+ "n_retries": 0,
+ "evals": {
+ "eval": {
+ "n_trials": 1,
+ "n_errors": 0,
+ "reward_stats": {"reward": {"0.8": ["build-case__attempt"]}},
+ }
+ },
+ },
+ }
+ )
+ )
+ result = collect_harbor_results(
+ "demo", ["opencode"], tmp_path / "output", jobs, expected_cases=1, expected_case_ids=["build-case"]
+ )
+ persisted = json.loads(json.dumps(result))
+ coverage = persisted["agents"]["opencode"]["declarative_checks"]
+ assert coverage["with_skill"]["passed_count"] == 1
+ assert coverage["with_skill"]["score_denominator"] == 4
+ assert coverage["without_skill"]["not_checked_count"] == 3
+ assert coverage["with_skill"]["results"][1]["reason"] == "Invoked but exited 2"
+
+
+def test_failed_arm_retains_coverage_without_restoring_withheld_quality(tmp_path):
+ arm = tmp_path / "run" / "agent-a" / "with-skill"
+ trial = arm / "trials" / "build-case__attempt"
+ trial.mkdir(parents=True)
+ (trial / "reward.json").write_text(json.dumps(_reward()))
+ (arm / "summary.json").write_text(
+ json.dumps(
+ {
+ "scores": dict.fromkeys(DEFAULT_METRICS, 0.8),
+ "metrics": list(DEFAULT_METRICS),
+ "num_trials": 1,
+ "execution_status": "failed",
+ "execution_errors": ["Another attempt failed"],
+ "expected_attempts": 2,
+ "scored_attempts": 1,
+ }
+ )
+ )
+ skill = tmp_path / "demo"
+ skill.mkdir()
+ result = agent_eval_result_from_directory(skill, tmp_path / "run", use_llm_judge=False)
+ agent = result.metadata["agent_eval"]["agents"]["agent-a"]
+ assert agent["declarative_checks"]["with_skill"]["score_denominator"] == 4
+ assert agent["declarative_checks"]["with_skill"]["results"][2]["passed"] is None
+ assert agent["with_skill"] is None
+ assert agent["conditions"]["with_skill"]["execution_status"] == "failed"
+ html = HTMLReporter(report_scope="tier3").render(result)
+ assert "behavior denominator 4" in html
+ assert "File evidence unavailable" in html
+
+
+def test_compact_coverage_keeps_exact_counts_and_prefers_failing_rows():
+ full = aggregate_declarative_checks([_reward() for _ in range(4)])
+ original = copy.deepcopy(full)
+ assert len(full["results"]) == 12
+
+ compact, dropped = compact_declarative_checks({"with_skill": full}, max_rows=5)
+
+ arm = compact["with_skill"]
+ assert dropped == 7
+ assert arm["omitted_checks"] == 7
+ assert [row["passed"] for row in arm["results"]] == [False, None, False, None, False]
+ assert {key: value for key, value in arm.items() if key not in {"results", "omitted_checks"}} == {
+ key: value for key, value in full.items() if key != "results"
+ }
+ assert full == original
+ assert compact_declarative_checks(compact, max_rows=5) == (compact, 0)
+ assert compact_declarative_checks(compact, max_rows=3)[0]["with_skill"]["omitted_checks"] == 9
+ assert compact_declarative_checks({"with_skill": full}, max_rows=12) == ({"with_skill": full}, 0)
+
+
+def test_coverage_summary_says_how_many_check_rows_were_omitted():
+ group = {"checked_count": 2, "passed_count": 1, "failed_count": 1, "judged_count": 0}
+ assert "omitted" not in coverage_summary(group)
+ assert coverage_summary({**group, "omitted_checks": 7}).endswith("; 7 more check rows omitted to fit report limits")
+
+
+def _large_run_info(*, cases, attempts, checks):
+ def rewards(arm):
+ return [
+ {
+ **dict.fromkeys(DEFAULT_METRICS, 0.8),
+ "entry_id": f"case-{case:03d}",
+ "_trial_name": f"case-{case:03d}__{arm}{attempt}",
+ "details": {
+ "behavior_check": {
+ "status": "accepted",
+ "declarative_checks": {
+ "judged_count": 0,
+ "results": [
+ {
+ "type": "script_invoked",
+ "expected": f"scripts/s{check}.py",
+ "passed": check != 0,
+ "reason": f"check {check}",
+ }
+ for check in range(checks)
+ ],
+ },
+ }
+ },
+ }
+ for case in range(cases)
+ for attempt in range(attempts)
+ ]
+
+ info = _info(checks=False)
+ info.update(rewards=rewards("a"), rewards_baseline=rewards("b"), without_skill=dict.fromkeys(DEFAULT_METRICS, 0.6))
+ return info
+
+
+def test_large_declarative_run_bounds_coverage_before_scored_report_details():
+ info = _large_run_info(cases=120, attempts=3, checks=10)
+ full = agent_declarative_checks(info["rewards"], info["rewards_baseline"])
+ assert all(len(arm["results"]) == 3600 for arm in full.values())
+
+ payload = build_agent_eval_payload("demo", {"agent-a": info}, use_llm_judge=False)
+
+ assert agent_eval_report_serialization_issue(payload) is None
+ agent = payload["agents"]["agent-a"]
+ for arm, expected in full.items():
+ coverage = agent["declarative_checks"][arm]
+ assert len(coverage["results"]) == COMPACT_CHECK_ROWS_PER_ARM
+ assert coverage["omitted_checks"] == 3600 - COMPACT_CHECK_ROWS_PER_ARM
+ assert {key: value for key, value in coverage.items() if key not in {"results", "omitted_checks"}} == {
+ key: value for key, value in expected.items() if key != "results"
+ }
+ omitted = payload["report_truncation"]["omitted"]
+ assert omitted["declarative_check_rows"] == 2 * (3600 - COMPACT_CHECK_ROWS_PER_ARM)
+ assert not {"non_best_agent_details", "dataset_and_trial_items", "attempt_verdict_details"} & set(omitted)
+ assert payload["overall_score"] == pytest.approx(0.8)
+ assert agent["with_skill"] == pytest.approx(0.8)
+ assert agent["baseline"] == pytest.approx(0.6)
+ assert agent["lift"] == pytest.approx(0.2)
+ assert len(agent["dimensions"]) == 5
+ assert len(agent["evaluator_cards"]) == 6
+ assert len(agent["trials"]) == len(agent["trials_baseline"]) == len(agent["cases"]) == 360
+ html = HTMLReporter(report_scope="tier3").render(_validation(payload))
+ assert "3400 more check rows omitted to fit report limits" in html
+
+
+def test_declarative_run_within_report_limits_keeps_every_check_row():
+ info = _large_run_info(cases=10, attempts=3, checks=10)
+ payload = build_agent_eval_payload("demo", {"agent-a": info}, use_llm_judge=False)
+ for arm in payload["agents"]["agent-a"]["declarative_checks"].values():
+ assert len(arm["results"]) == 300
+ assert "omitted_checks" not in arm
+ assert "declarative_check_rows" not in payload.get("report_truncation", {}).get("omitted", {})
diff --git a/tests/test_harbor_runner_artifacts.py b/tests/test_harbor_runner_artifacts.py
index 6c8786cb..73b47a7b 100644
--- a/tests/test_harbor_runner_artifacts.py
+++ b/tests/test_harbor_runner_artifacts.py
@@ -13,6 +13,7 @@ import pytest
import skillevaluator.tier3.harbor.runner as harbor_runner
from skillevaluator import __version__
+from skillevaluator.tier3.declarative_coverage import COMPACT_CHECK_ROWS_PER_ARM, agent_declarative_checks
from skillevaluator.tier3.harbor import collector as harbor_collector
from skillevaluator.tier3.harbor import report_data
@@ -580,6 +581,94 @@ def test_final_result_fails_closed_before_publishing_unreferenced_oversize_detai
assert not list(tmp_path.glob(".result.json.*"))
+def _declarative_rewards(arm: str, *, cases: int, attempts: int, checks: int) -> list[dict[str, Any]]:
+ return [
+ {
+ "entry_id": f"case-{case:03d}",
+ "_trial_name": f"case-{case:03d}__{arm}{attempt}",
+ "overall": 0.8,
+ "details": {
+ "behavior_check": {
+ "status": "accepted",
+ "declarative_checks": {
+ "judged_count": 1,
+ "results": [
+ {
+ "type": "script_invoked",
+ "expected": f"scripts/s{check}.py",
+ "passed": check != 0,
+ "reason": f"check {check}",
+ }
+ for check in range(checks)
+ ],
+ },
+ }
+ },
+ }
+ for case in range(cases)
+ for attempt in range(attempts)
+ ]
+
+
+def test_final_result_publishes_a_large_declarative_run_with_bounded_coverage(tmp_path: Path) -> None:
+ rewards = {
+ arm: _declarative_rewards(arm, cases=120, attempts=3, checks=10) for arm in ("with-skill", "without-skill")
+ }
+ coverage = agent_declarative_checks(rewards["with-skill"], rewards["without-skill"])
+ pass_summary = {"pass_at_k": {"1": 0.8}, "cases": {}, "extra_cases": []}
+ agent = {
+ "with_skill": {"overall": 0.8},
+ "without_skill": {"overall": 0.5},
+ "overall_with_skill": 0.8,
+ "overall_without_skill": 0.5,
+ "lift": {"overall": 0.3},
+ "pass_at_k": {"with_skill": pass_summary, "without_skill": pass_summary, "lift": {"overall": 0.3}},
+ "execution_status": "succeeded",
+ "expected_attempts": 720,
+ "scored_attempts": 720,
+ "declarative_checks": coverage,
+ }
+ result: dict[str, Any] = {"execution_status": "succeeded", "execution_errors": [], "agents": {"opencode": agent}}
+ assert all(len(arm["results"]) == 3600 for arm in coverage.values())
+ with pytest.raises(harbor_runner._FinalResultContractError):
+ harbor_runner._validate_final_result_tree(result)
+
+ result_path = tmp_path / "result.json"
+ harbor_runner._write_final_result(result_path, result)
+
+ persisted = json.loads(result_path.read_text(encoding="utf-8"))
+ harbor_runner._validate_final_result_tree(persisted)
+ assert result_path.stat().st_size <= harbor_runner.FINAL_RESULT_MAX_BYTES
+ published = persisted["agents"]["opencode"]
+ for key in (
+ "with_skill",
+ "without_skill",
+ "overall_with_skill",
+ "overall_without_skill",
+ "lift",
+ "pass_at_k",
+ "execution_status",
+ "scored_attempts",
+ ):
+ assert published[key] == agent[key]
+ for arm, full in coverage.items():
+ compact = published["declarative_checks"][arm]
+ assert len(compact["results"]) == COMPACT_CHECK_ROWS_PER_ARM
+ assert compact["omitted_checks"] == len(full["results"]) - COMPACT_CHECK_ROWS_PER_ARM
+ assert all(row["passed"] is False for row in compact["results"])
+ assert {key: value for key, value in compact.items() if key not in {"results", "omitted_checks"}} == {
+ key: value for key, value in full.items() if key != "results"
+ }
+ # 360 rewards, each with 9 passed, 1 failed, and 1 judged requirement.
+ assert published["declarative_checks"]["with_skill"]["failed_count"] == 360
+ assert published["declarative_checks"]["with_skill"]["score_denominator"] == 360 * (9 + 1 + 1)
+ assert persisted["result_projection"]["omitted_detail_fields"]["opencode"] == [
+ "declarative_checks.with_skill.results",
+ "declarative_checks.without_skill.results",
+ ]
+ assert all(len(arm["results"]) == 3600 and "omitted_checks" not in arm for arm in coverage.values())
+
+
def test_final_result_projects_real_multi_agent_aggregate_errors_without_losing_exact_total(tmp_path: Path) -> None:
run_dir = tmp_path / "run"
run_dir.mkdir()