Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""CLI reporter using Rich for terminal output.
This reporter provides colorful, formatted terminal output using the Rich
library. It's the default reporter for interactive use.
Features:
- Colored output with severity-based highlighting
- Summary table for multiple validators
- Tree-structured findings display
- Progress indicators and spinners
Text that SkillEvaluator does not author (skill and plugin content, paths,
LLM judge output, external tool messages) goes through ``escape_markup`` before
it is interpolated into Rich markup: ``[/x]`` would otherwise raise
``rich.errors.MarkupError`` and ``[link=...]`` could restyle or spoof output.
Consoles are created with ``emoji=False`` so ``:name:`` codes in that text
(``root:x:0:0:``) print literally.
"""
from __future__ import annotations
import math
from io import StringIO
from typing import TYPE_CHECKING
from rich.console import Console
from rich.panel import Panel
from rich.table import Table
from rich.text import Text
from skillevaluator.constants import (
DIMENSION_VERDICT_NEUTRAL_THRESHOLD,
DIMENSION_VERDICT_PASS_THRESHOLD,
)
from skillevaluator.reporting.base import (
ReporterBase,
_agent_eval_finite_number,
additional_errors,
additional_warnings,
get_skip_reason,
is_advisory_agent_eval_skip,
is_cleanly_skipped,
passes_required_gate,
source_validation_errors,
source_validation_preview,
)
from skillevaluator.reporting.declarative_coverage import render_declarative_coverage
from skillevaluator.reporting.finding_verification import finding_verification
from skillevaluator.reporting.harbor_viewer import (
harbor_evidence_link_text,
harbor_job_links,
safe_url,
)
from skillevaluator.reporting.presentation import FinalRenderContext, build_final_render_context, trial_coverage
from skillevaluator.reporting.run_status import Status, result_status
from skillevaluator.reporting.summary_copy import SKILL_LIFT_DEFINITION, pass_at_k_summary
from skillevaluator.reporting.terminal_styles import AMBER, FAINT, GREEN, RED, TEXT, grade_style, lift_style
from skillevaluator.tier3.baseline_store import baseline_description
from skillevaluator.tier3.dataset_provenance import dataset_provenance_display
from skillevaluator.tier3.evidence_volume import format_evidence_volume
from skillevaluator.tier3.version_comparison import version_comparison_lines
from skillevaluator.utils.rich_markup import escape_markup
if TYPE_CHECKING:
from collections.abc import Iterable
from skillevaluator.models import Finding, ValidationResult
_SKIP_REASON_MAX_CHARS = 1024
_SOURCE_ERROR_MAX_CHARS = 1024
def _related_paths(finding: Finding) -> list[str]:
"""Return distinct path-like string values carried in finding metadata."""
metadata = finding.metadata if isinstance(finding.metadata, dict) else {}
paths: list[str] = []
for key, value in metadata.items():
normalized_key = str(key).casefold()
if not (normalized_key == "path" or normalized_key.startswith("path_") or normalized_key.endswith("_path")):
continue
if isinstance(value, str) and value and value not in paths:
paths.append(value)
return paths
def _rich_source_error(error: dict[str, str]) -> str:
"""Return one failed Tier 3 source check as a bounded line inert to Rich markup."""
text = f"{error['path']}: {error['message']}" if error["path"] else error["message"]
text = " ".join(text.split())
if len(text) > _SOURCE_ERROR_MAX_CHARS:
text = text[: _SOURCE_ERROR_MAX_CHARS - 1] + "…"
return escape_markup(text)
def _rich_line(text: str) -> str:
"""Return one bounded, single-line message inert to Rich markup."""
line = " ".join(text.replace("\r\n", "\n").replace("\r", "\n").split())
if len(line) > _SKIP_REASON_MAX_CHARS:
line = line[: _SKIP_REASON_MAX_CHARS - 1] + "…"
return escape_markup(line)
def _rich_skip_reason(result: ValidationResult) -> str:
"""Return one bounded, single-line skip reason inert to Rich markup."""
return _rich_line(get_skip_reason(result))
def _coverage_notice_messages(agent_eval: object) -> list[str]:
"""Return the Tier 3 coverage notice messages the Tier 3 block prints."""
notices = agent_eval.get("coverage_notices") if isinstance(agent_eval, dict) else None
return [
notice["message"]
for notice in (notices if isinstance(notices, list) else [])
if isinstance(notice, dict) and isinstance(notice.get("message"), str) and notice["message"]
]
class CLIReporter(ReporterBase):
"""Terminal-based reporter with Rich formatting.
Provides colorful, structured output for terminal/console display.
Supports both direct printing and string capture for testing.
"""
def __init__(
self,
console: Console | None = None,
*,
report_formats: Iterable[str] = (),
presentation_context: FinalRenderContext | None = None,
) -> None:
"""Initialize CLI reporter.
Args:
console: Rich Console instance (creates new one if not provided)
report_formats: Every format the run emits alongside this report.
Failed Tier 3 source checks are cut to a preview only when one
of these files lists all of them.
"""
self._console = console
self._report_formats = tuple(report_formats)
self._presentation_context = presentation_context
@property
def console(self) -> Console:
"""Get or create the Rich console instance."""
if self._console is None:
self._console = Console(emoji=False)
return self._console
@property
def name(self) -> str:
return "cli"
@property
def description(self) -> str:
return "Terminal output with Rich formatting"
def render(self, result: ValidationResult) -> str:
"""Render single result to string (captures console output)."""
string_io = StringIO()
temp_console = Console(file=string_io, force_terminal=True, emoji=False)
self.render_result(result, temp_console)
return string_io.getvalue()
def render_all(self, results: list[ValidationResult]) -> str:
"""Render all results to string with summary table."""
string_io = StringIO()
temp_console = Console(file=string_io, force_terminal=True, emoji=False)
self._render_all_results(results, temp_console)
return string_io.getvalue()
def print(self, result: ValidationResult) -> None:
"""Print single result directly to console."""
self.render_result(result, self.console)
def print_all(self, results: list[ValidationResult]) -> None:
"""Print all results directly to console."""
self._render_all_results(results, self.console)
def print_summary(self, results: list[ValidationResult]) -> None:
"""Print only the summary table (no failure details, no overall verdict).
Used for progressive/interim CLI output -- e.g. flushing Tier 1 and
Tier 2 results to the terminal before the long-running Tier 3 agent
evaluation, so they stay visible in CI logs even when Tier 3 is slow,
errors, or is interrupted before the final combined report is emitted.
"""
self._print_summary_table(results, self.console)
self.console.print()
def _render_all_results(self, results: list[ValidationResult], console: Console) -> None:
"""Render all results with summary table and failure details."""
context = self._presentation_context or build_final_render_context(results)
# Print summary table
self._print_summary_table(results, console, presentation=context)
console.print()
failed = [result for result in results if not passes_required_gate(result)]
# A non-blocking Tier 3 result (the ``validate`` default) can fail with
# no findings; its coverage notes still say which agent arm failed.
non_blocking = [
result
for result in results
if passes_required_gate(result)
and (result.findings or _coverage_notice_messages((result.metadata or {}).get("agent_eval")))
]
rendered_result_ids = {id(result) for result in (*failed, *non_blocking)}
for result in results:
metadata = result.metadata if isinstance(result.metadata, dict) else {}
agent_eval = metadata.get("agent_eval")
if (
result.validator_name == "AGENT_EVAL"
and id(result) not in rendered_result_ids
and not self._is_advisory_agent_eval_skip(result)
and isinstance(agent_eval, dict)
and result_status(result) not in {Status.SKIPPED, Status.INCOMPLETE}
):
self._print_agent_eval_tables(agent_eval, console)
rendered_result_ids.add(id(result))
# Print detailed results for failures
if failed:
console.print(
Panel.fit(
"[bold]Failure Details[/bold]",
style="yellow",
border_style="yellow",
)
)
for result in failed:
self.render_result(result, console)
if non_blocking:
console.print(
Panel.fit(
"[bold]Non-blocking Findings[/bold]",
style="yellow",
border_style="yellow",
)
)
for result in non_blocking:
self.render_result(result, console)
# Successful results do not enter the detailed failure/finding sections.
rendered = rendered_result_ids
for result in results:
if id(result) not in rendered:
agent_eval = result.metadata.get("agent_eval")
if isinstance(agent_eval, dict):
provenance_line, provenance_is_template = dataset_provenance_display(
agent_eval.get("dataset_provenance")
)
if provenance_line:
console.print(Text(f" {provenance_line}", style="yellow" if provenance_is_template else "dim"))
self._print_harbor_artifacts(agent_eval, console)
color = {Status.PASSED: "green", Status.FAILED: "red"}.get(context.command_status, "yellow")
console.print(f"\n[bold {color}][{context.command_status.verdict}] {context.command_label}[/bold {color}]")
for number, tier in context.tiers.items():
if not tier.displayed:
continue
console.print(f"Tier {number}: {tier.summary}", highlight=False, soft_wrap=True)
if context.tiers[3].status not in (Status.NOT_RUN, Status.SKIPPED):
console.print(context.coverage.text, highlight=False, soft_wrap=True)
for line in context.coverage.details:
console.print(line, highlight=False, soft_wrap=True)
console.print(f"Publication eligibility: {context.publication_label}\n")
def render_result(self, result: ValidationResult, console: Console) -> None:
"""Render a single validation result."""
# Header
console.print(f"\n[bold]{escape_markup(f'[{result.validator_name}]')}[/bold]")
if result.validator_description:
console.print(f"[dim]{escape_markup(str(result.validator_description))}[/dim]")
# Quality score display (unified table for single and multi-skill)
qs = result.metadata.get("quality_scores") if result.metadata else None
qs_all = result.metadata.get("quality_scores_all") if result.metadata else None
if qs and qs.get("dimensions"):
grade_colors = {grade: grade_style(grade, status=result_status(result).value) for grade in "ABCDF"}
skills_list = qs_all or [qs]
self._print_quality_table(qs, skills_list, grade_colors, console)
# LLM rubric evaluation display
rubric = result.metadata.get("rubric_eval") if result.metadata else None
if rubric and rubric.get("checks"):
self._print_rubric_table(rubric, console)
# Tier 3: Agent evaluation display
agent_eval = result.metadata.get("agent_eval") if result.metadata else None
if agent_eval:
self._print_agent_eval_tables(agent_eval, console)
if result_status(result) == Status.SKIPPED:
label = (
"Live evaluation did not run"
if self._is_advisory_agent_eval_skip(result)
else "Validation check skipped"
)
console.print(f"[yellow][SKIP] {label}[/yellow]\n")
self._print_summary_stats(result, console)
elif result_status(result) == Status.INCOMPLETE:
tools = ", ".join(result.incomplete_scans)
console.print(f"[yellow][INCOMPLETE] {escape_markup(tools)} did not complete[/yellow]\n")
self._print_reference_summary(result, console)
self._print_summary_stats(result, console)
self._print_findings(result, console)
elif result.passed:
console.print("[green][PASS] Validation passed[/green]\n")
self._print_summary_stats(result, console)
self._print_success_details(result, console)
if result.findings:
self._print_findings(result, console)
else:
console.print("[red][FAIL] Validation failed[/red]\n")
self._print_reference_summary(result, console)
self._print_summary_stats(result, console)
self._print_findings(result, console)
@staticmethod
def _print_reference_summary(result: ValidationResult, console: Console) -> None:
"""Print how many referenced artifacts SkillSpector did not completely inspect in a failed result."""
from skillevaluator.skillspector_references import reference_summary_message
summary = reference_summary_message(result.metadata.get("skillspector_reference_summary"))
if summary and not result.passed:
console.print(f" {escape_markup(summary)}", soft_wrap=True, highlight=False)
console.print()
@staticmethod
def _score_cell(dims: dict, dname: str) -> str:
"""Format a dimension score cell with color coding."""
d = dims.get(dname, {})
s = d.get("score", 0)
c = GREEN if s >= 80 else (AMBER if s >= 60 else RED)
return f"[{c}]{s:.0f}[/{c}]"
@staticmethod
def _print_quality_table(
qs: dict,
skills_list: list[dict],
grade_colors: dict[str, str],
console: Console,
) -> None:
"""Print a unified quality scores table for single or multi-skill runs."""
multi = len(skills_list) > 1
avg_score = qs.get("overall_score", 0)
avg_grade = qs.get("grade", "?")
gc = grade_colors.get(avg_grade, "white")
avg_grade_text = escape_markup(str(avg_grade))
if multi:
console.print(f"\n [{gc}]Average: {avg_score:.1f}/100 (Grade: {avg_grade_text})[/{gc}]")
skill_count = escape_markup(str(qs.get("skill_count", len(skills_list))))
console.print(f" [dim]Skills analyzed:[/dim] {skill_count}\n")
else:
stype = escape_markup(str(skills_list[0].get("skill_type", "unknown")))
console.print(f"\n [{gc}]Overall: {avg_score:.1f}/100 (Grade: {avg_grade_text})[/{gc}]")
console.print(f" [dim]Skill Type:[/dim] {stype}\n")
table = Table(title="Quality Scores by Skill", border_style=FAINT, show_header=True)
table.add_column("Skill", style="bold")
table.add_column("Grade", justify="center")
table.add_column("Score", justify="right")
table.add_column("Correctness", justify="right")
table.add_column("Discoverability", justify="right")
table.add_column("Reliability", justify="right")
table.add_column("Efficiency", justify="right")
for skill_qs in skills_list:
sname = skill_qs.get("skill_name", "?")
sgrade = skill_qs.get("grade", "?")
sscore = skill_qs.get("overall_score", 0)
sgc = grade_colors.get(sgrade, "white")
dims = skill_qs.get("dimensions", {})
table.add_row(
escape_markup(str(sname)),
f"[{sgc}]{escape_markup(str(sgrade))}[/{sgc}]",
f"[{sgc}]{sscore:.1f}[/{sgc}]",
CLIReporter._score_cell(dims, "correctness"),
CLIReporter._score_cell(dims, "discoverability"),
CLIReporter._score_cell(dims, "reliability"),
CLIReporter._score_cell(dims, "efficiency"),
)
if multi:
table.add_section()
agc = grade_colors.get(avg_grade, "white")
avg_dims = qs.get("dimensions", {})
table.add_row(
"[bold]Average[/bold]",
f"[{agc}]{avg_grade_text}[/{agc}]",
f"[{agc}]{avg_score:.1f}[/{agc}]",
CLIReporter._score_cell(avg_dims, "correctness"),
CLIReporter._score_cell(avg_dims, "discoverability"),
CLIReporter._score_cell(avg_dims, "reliability"),
CLIReporter._score_cell(avg_dims, "efficiency"),
)
console.print(table)
console.print()
@staticmethod
def _print_rubric_table(rubric: dict, console: Console) -> None:
"""Print LLM rubric evaluation results as a table."""
score = rubric.get("overall_score", 0)
color = "green" if score >= 80 else ("yellow" if score >= 60 else "red")
console.print(f"\n [{color}]LLM Rubric Score: {score}/100[/{color}]")
summary = rubric.get("summary", "")
if summary:
console.print(f" [dim]{escape_markup(str(summary))}[/dim]")
console.print()
table = Table(title="Rubric Evaluation", border_style=FAINT, show_header=True)
table.add_column("Criterion", style="bold")
table.add_column("Score", justify="center")
table.add_column("Pass", justify="center")
table.add_column("Notes")
for check in rubric.get("checks", []):
cs = check.get("score", 0)
cc = "green" if cs >= 7 else ("yellow" if cs >= 5 else "red")
passed = "[green]Yes[/green]" if check.get("pass") else "[red]No[/red]"
table.add_row(
escape_markup(str(check.get("id", "?")).replace("_", " ").title()),
f"[{cc}]{cs}/10[/{cc}]",
passed,
escape_markup(str(check.get("notes") or "")),
)
console.print(table)
console.print()
@staticmethod
def _print_harbor_artifacts(agent_eval: dict, console: Console) -> None:
jobs = harbor_job_links(agent_eval)
if jobs:
console.print(" [bold]Harbor artifacts:[/bold]")
for job in jobs:
console.print(f" [bold]{escape_markup(job['name'])}[/bold]", highlight=False)
if job.get("url"):
console.print(f" Harbor logs: Results {escape_markup(job['url'])}", soft_wrap=True, highlight=False)
if job.get("analysis_url"):
console.print(
f" Harbor analysis: {escape_markup(job['analysis_url'])}", soft_wrap=True, highlight=False
)
@staticmethod
def _print_agent_eval_tables(agent_eval: dict, console: Console) -> None:
"""Print Tier 3 agent evaluation results."""
render_declarative_coverage(console, agent_eval.get("agents"))
if agent_eval.get("baseline"):
console.print(escape_markup(baseline_description(agent_eval["baseline"])))
verdict = agent_eval.get("verdict", "unknown")
best_agent = agent_eval.get("best_agent")
multiple_agents = isinstance(agent_eval.get("agents_run"), list) and len(agent_eval["agents_run"]) > 1
composite = agent_eval.get("composite_lift")
runtime = agent_eval.get("runtime_seconds", 0.0)
vc = GREEN if verdict == "pass" else (RED if verdict == "fail" else AMBER)
scored_lift = (
isinstance(composite, int | float) and not isinstance(composite, bool) and math.isfinite(composite)
)
composite_text = f"{composite:+.2f}" if scored_lift else "N/A"
composite_style = lift_style(composite) if scored_lift else "dim"
verdict_text = escape_markup(str(verdict).upper())
verdict_reason = agent_eval.get("verdict_reason")
safe_reason = (
f" — {escape_markup(str(verdict_reason))}" if verdict in {"fail", "incomplete"} and verdict_reason else ""
)
budget = agent_eval.get("budget")
if isinstance(budget, dict):
from skillevaluator.tier3.harbor.run_budget import budget_line
console.print(escape_markup(budget_line(budget)))
lift_label = (
"Skill Lift (for reference)" if verdict == "fail" and scored_lift and composite > 0 else "Skill Lift"
)
console.print(
f"\n [{vc}]Verdict: {verdict_text}[/{vc}]{safe_reason} "
f"({lift_label} = [{composite_style}]{composite_text}[/{composite_style}])"
)
if scored_lift:
console.print(Text(SKILL_LIFT_DEFINITION, style="dim"), soft_wrap=True)
if pass_summary := pass_at_k_summary(agent_eval.get("pass_at_k"), agent_eval.get("attempt_policy")):
console.print(f" [dim]{pass_summary}[/dim]")
for label, count_data, numerator in (
("Required checks passed", agent_eval.get("required_checks"), "passed"),
("Goals met", agent_eval.get("correct_answers"), "correct"),
("Security clean", agent_eval.get("security_clean"), "clean"),
):
with_skill = count_data.get("with_skill") if isinstance(count_data, dict) else None
without_skill = count_data.get("without_skill") if isinstance(count_data, dict) else None
difference = count_data.get("difference") if isinstance(count_data, dict) else None
with_counts = (with_skill.get(numerator), with_skill.get("scored")) if isinstance(with_skill, dict) else ()
without_counts = (
(without_skill.get(numerator), without_skill.get("scored")) if isinstance(without_skill, dict) else ()
)
valid_with = (
len(with_counts) == 2
and all(isinstance(count, int) and not isinstance(count, bool) and count >= 0 for count in with_counts)
and with_counts[1] > 0
)
valid_without = (
len(without_counts) == 2
and all(
isinstance(count, int) and not isinstance(count, bool) and count >= 0 for count in without_counts
)
and without_counts[1] > 0
)
if valid_with:
suffix = f" ({best_agent})" if multiple_agents and best_agent else ""
line = f" {label}{suffix}: {with_counts[0]}/{with_counts[1]}"
if valid_without:
if isinstance(difference, int) and not isinstance(difference, bool):
delta = f"{difference:+d}"
elif isinstance(difference, int | float) and not isinstance(difference, bool):
delta = f"{difference:+.1f} pp"
else:
delta = "N/A"
line += f" | {without_counts[0]}/{without_counts[1]} | {delta}"
else:
line += " | N/A | N/A"
console.print(line)
provenance_line, provenance_is_template = dataset_provenance_display(agent_eval.get("dataset_provenance"))
if provenance_line:
console.print(Text(f" {provenance_line}", style="yellow" if provenance_is_template else "dim"))
if verdict == "inconclusive":
console.print(
" [yellow]Skill never activated. Scores are diagnostic only; no lift verdict was computed.[/yellow]"
)
if runtime:
console.print(f" [dim]Runtime: {runtime:.1f}s[/dim]")
evidence_line = format_evidence_volume(agent_eval.get("evidence_volume"))
if evidence_line:
style = "yellow" if evidence_line.startswith("Limited evidence:") else "dim"
console.print(f" [{style}]{evidence_line}[/{style}]")
sufficiency = agent_eval.get("dataset_sufficiency")
if isinstance(sufficiency, dict):
status = str(sufficiency.get("status") or "provisional")
style = "yellow" if status == "provisional" else "green"
line = (
sufficiency.get("coverage_summary")
or f"Dataset sufficiency: {status} — {sufficiency.get('reason', '')}"
)
console.print(f" [{style}]{line}[/{style}]")
CLIReporter._print_harbor_artifacts(agent_eval, console)
pass_at_k = agent_eval.get("pass_at_k")
with_skill_pass = pass_at_k.get("with_skill") if isinstance(pass_at_k, dict) else None
case_arm_label = "with skill"
case_summaries = with_skill_pass.get("cases") if isinstance(with_skill_pass, dict) else None
if not isinstance(case_summaries, dict) or not case_summaries:
without_skill_pass = pass_at_k.get("without_skill") if isinstance(pass_at_k, dict) else None
case_summaries = without_skill_pass.get("cases") if isinstance(without_skill_pass, dict) else None
case_arm_label = "without skill"
if isinstance(case_summaries, dict) and case_summaries:
title = "Tier 3 Cases" if case_arm_label == "with skill" else "Tier 3 Cases (without skill)"
table = Table(title=title, border_style=FAINT, show_header=True)
table.add_column("Case", style="bold")
table.add_column("Score", justify="right")
table.add_column("Result")
table.add_column("Warnings")
for case_id, case in case_summaries.items():
if not isinstance(case, dict):
continue
score = case.get("best_score")
score_text = f"{score:.2f}" if isinstance(score, int | float) and not isinstance(score, bool) else "N/A"
if case.get("passed") is True:
result_text = "[green]PASS[/green]"
elif (
isinstance(case.get("attempts_used"), int)
and not isinstance(case.get("attempts_used"), bool)
and case.get("attempts_used") > 0
):
result_text = "[red]FAIL[/red]"
reason = case.get("reason")
if isinstance(reason, str) and reason:
result_text += f" — {escape_markup(reason)}"
else:
result_text = "[yellow]not scored[/yellow]"
warnings = case.get("warnings")
warning_text = (
"; ".join(str(item) for item in warnings if isinstance(item, str))
if isinstance(warnings, list)
else ""
)
table.add_row(
escape_markup(str(case_id)),
score_text,
result_text,
f"[yellow]WARNING[/yellow] {escape_markup(warning_text)}" if warning_text else "",
)
if table.rows:
console.print(table)
console.print()
console.print(f" {trial_coverage(agent_eval).text}")
console.print()
evaluators = agent_eval.get("evaluators", {})
if evaluators:
table = Table(
title="Evaluator Scores (Skill Lift)",
border_style=FAINT,
show_header=True,
)
table.add_column("Evaluator", style="bold")
table.add_column("With Skill", justify="right")
table.add_column("Baseline", justify="right")
table.add_column("Lift", justify="right")
for name, scores in evaluators.items():
ws = scores.get("with_skill")
bl = scores.get("baseline")
lift = scores.get("lift")
# A baseline-only run (--skip-with-skill) never ran the with-skill arm.
with_skipped = scores.get("with_skill_status") == "skipped"
if lift is None:
lift_text = (
"N/A"
if "not_applicable" in (scores.get("with_skill_status"), scores.get("baseline_status"))
else "skipped"
if with_skipped
else "NO SCORE"
)
else:
lc = lift_style(lift)
lift_text = f"[{lc}]{lift:+.2f}[/{lc}]"
table.add_row(
escape_markup(str(name).replace("_", " ").title()),
f"{ws:.2f}"
if ws is not None
else "N/A"
if scores.get("with_skill_status") == "not_applicable"
else "skipped"
if with_skipped
else "NO SCORE",
f"{bl:.2f}"
if bl is not None
else "N/A"
if scores.get("baseline_status") == "not_applicable"
else "NO SCORE",
lift_text,
)
console.print(table)
console.print()
# Name arms that ran but failed, so a missing baseline or score is not read as an arm that never ran.
coverage_messages = _coverage_notice_messages(agent_eval)
if coverage_messages:
console.print("[bold]Agent coverage notes[/bold]")
for message in coverage_messages:
console.print(f" • {escape_markup(message)}", soft_wrap=True)
console.print()
from skillevaluator.reporting.tier3_token_usage import VALIDATE_CASE_POINTER, render_token_usage_table
if render_token_usage_table(console, agent_eval.get("token_usage"), case_pointer=VALIDATE_CASE_POINTER):
console.print()
recommendations = agent_eval.get("recommendations") or []
if recommendations:
printed = False
for recommendation in recommendations[:5]:
if not isinstance(recommendation, dict):
continue
message = str(recommendation.get("message") or recommendation.get("title") or "").strip()
if not message:
continue
if not printed:
console.print("[bold]Recommendations[/bold]")
printed = True
console.print(f" • {escape_markup(message)}", soft_wrap=True)
evidence = recommendation.get("evidence")
if isinstance(evidence, dict):
url = safe_url(evidence.get("url"))
if url:
link_text = escape_markup(harbor_evidence_link_text(evidence))
console.print(f" [dim]{link_text}:[/dim] [dim]{escape_markup(url)}[/dim]", soft_wrap=True)
if printed:
console.print()
insights = agent_eval.get("insights", {})
if any(v.get("score") is not None for v in insights.values()):
table = Table(
title="LLM-as-Judge Insights",
border_style=FAINT,
show_header=True,
)
table.add_column("Dimension", style="bold")
table.add_column("Score", justify="center")
table.add_column("Explanation")
for dim, info in insights.items():
score = info.get("score")
if score is None:
continue
explanation = info.get("explanation", "")
if isinstance(score, str):
sc = "green" if score.upper() == "PASS" else "red"
score_str = f"[{sc}]{escape_markup(score)}[/{sc}]"
else:
sc = (
"green"
if score >= DIMENSION_VERDICT_PASS_THRESHOLD
else ("yellow" if score >= DIMENSION_VERDICT_NEUTRAL_THRESHOLD else "red")
)
score_str = f"[{sc}]{score:.2f}[/{sc}]"
# Truncate before escaping so the cut cannot split an escape sequence.
table.add_row(
escape_markup(str(dim).title()),
score_str,
escape_markup(str(explanation or "")[:80]),
)
console.print(table)
console.print()
def _print_summary_stats(self, result: ValidationResult, console: Console) -> None:
"""Print summary statistics."""
s = result.summary
console.print("[dim]Summary:[/dim]")
if s.files_scanned > 0:
console.print(f" • Files scanned: {s.files_scanned}")
if s.checks_performed > 0:
console.print(f" • Checks performed: {s.checks_performed}")
if not result.passed:
error_detail = f"{s.errors}"
if s.critical_count > 0 or s.high_count > 0:
parts = []
if s.critical_count > 0:
parts.append(f"{s.critical_count} critical")
if s.high_count > 0:
parts.append(f"{s.high_count} high")
error_detail += f" ({', '.join(parts)})"
console.print(f" • Errors: {error_detail}")
if s.warnings > 0:
console.print(f" • Warnings: {s.warnings}")
console.print()
def _print_success_details(self, result: ValidationResult, console: Console) -> None:
"""Print success details."""
if not result.success_details:
# Fall back to legacy messages
if result.messages:
console.print("[dim]Details:[/dim]")
for msg in result.messages:
console.print(f" {escape_markup(str(msg))}")
return
console.print("[dim]Details:[/dim]")
for detail in result.success_details:
meta_str = ""
if detail.metadata:
meta_parts = [f"{k}={v}" for k, v in detail.metadata.items()]
if meta_parts:
meta_str = f" ({', '.join(meta_parts)})"
console.print(
" [green][OK][/green] "
f"{escape_markup(str(detail.check_name))}: "
f"{escape_markup(str(detail.message))}{escape_markup(meta_str)}",
highlight=False,
)
def _print_findings(self, result: ValidationResult, console: Console) -> None:
"""Print findings with tree structure."""
errors = additional_errors(result)
if errors:
label = "Execution errors" if result.is_incomplete else "Errors"
console.print(f"[dim]{label}:[/dim]")
for error in errors:
console.print(f" [red]•[/red] {escape_markup(str(error))}")
if not result.findings:
# Preserve legacy warnings when there are no structured findings.
if result.warnings:
console.print("[dim]Warnings:[/dim]")
for warning in result.warnings:
console.print(f" [yellow][WARN][/yellow] {escape_markup(str(warning))}")
return
# Warnings no finding repeats, such as why a scan is incomplete.
warnings = additional_warnings(result)
if warnings:
if errors:
console.print()
console.print("[dim]Warnings:[/dim]")
for warning in warnings:
console.print(f" [yellow][WARN][/yellow] {escape_markup(str(warning))}")
if errors or warnings:
console.print()
console.print("[dim]Issues:[/dim]")
for i, finding in enumerate(result.findings, 1):
self._print_finding(i, finding, console)
def _print_finding(self, index: int, finding: Finding, console: Console) -> None:
"""Print a single finding with structured details.
Dynamic finding fields (message, location, content, suggestion) are
escaped so values containing ``[...]`` are not parsed as Rich markup
(which would raise ``rich.errors.MarkupError`` and abort the report).
"""
severity_color = finding.severity.color
console.print(
f" {index}. [{severity_color}]{escape_markup(finding.tag)}[/{severity_color}] {escape_markup(finding.message)}"
)
console.print(f" [dim]File:[/dim] {escape_markup(str(finding.location))}")
console.print(f" [dim]Check:[/dim] {escape_markup(str(finding.check_name))}")
verification = finding_verification(finding)
if verification:
label = f"{verification.verdict} ({verification.confidence} confidence)"
console.print(f" [dim]LLM verification:[/dim] {escape_markup(label)}")
if verification.reasoning:
console.print(f" [dim]Reasoning:[/dim] {escape_markup(verification.reasoning)}")
console.print(f" [dim]{escape_markup(verification.severity_effect)}[/dim]")
related_paths = _related_paths(finding)
if related_paths:
console.print(f" [dim]Related paths:[/dim] {escape_markup(' <-> '.join(related_paths))}")
if finding.line_content:
content = finding.line_content.strip()
if len(content) > 70:
content = content[:67] + "..."
console.print(f" [dim]Content:[/dim] [italic]{escape_markup(content)}[/italic]")
if finding.suggestion:
label = (
"Recommendation" if finding.category in ("WHOLE_DUPLICATE", "PARTIAL_OVERLAP", "DUPLICATE") else "Fix"
)
console.print(f" [dim]{label}:[/dim] {escape_markup(finding.suggestion)}")
source_errors = source_validation_errors(finding.metadata)
if source_errors:
shown, overflow_note = source_validation_preview(source_errors, self._report_formats)
console.print(" [dim]Source validation:[/dim]")
for error in shown:
console.print(f" {_rich_source_error(error)}")
if overflow_note:
console.print(f" [dim]{escape_markup(overflow_note)}[/dim]")
console.print()
def _print_summary_table(
self, results: list[ValidationResult], console: Console, *, presentation: FinalRenderContext | None = None
) -> None:
"""Print summary table for all validators."""
table = Table(
title="SkillEvaluator Validation Results",
show_header=True,
header_style="bold white on dark_green",
border_style="green",
title_style="bold green",
)
table.add_column("Validator", style=TEXT)
table.add_column("Status", justify="center")
table.add_column("Details")
for result in results:
outcome = presentation.result_statuses[id(result)] if presentation else result_status(result).value
advisory_skip = outcome == "skipped"
status = {
"skipped": "[yellow]SKIP[/yellow]",
"not_applicable": "[dim]N/A[/dim]",
"incomplete": "[bold yellow]INCOMPLETE[/bold yellow]",
"passed": "[green]PASS[/green]",
"pass": "[green]PASS[/green]",
}.get(outcome, "[red]FAIL[/red]")
s = result.summary
static_test_evidence = self._static_test_evidence_message(result)
if advisory_skip or (outcome == "not_applicable" and is_cleanly_skipped(result)):
details = _rich_skip_reason(result)
elif outcome == "not_applicable":
# Keep the recorded reason; "N/A │ 1 checks passed" contradicts itself.
details = _rich_line(result.success_details[0].message) if result.success_details else "Not applicable"
elif outcome == "incomplete":
scanners = escape_markup(", ".join(result.incomplete_scans))
details = f"[bold yellow]{scanners} did not complete[/bold yellow]"
counts = []
if s.errors:
counts.append(f"{s.errors} errors")
if s.warnings:
counts.append(f"{s.warnings} warnings")
if counts:
details += f" ({', '.join(counts)})"
elif result.passed:
if result.metadata.get("skipped"):
details = "Skipped (see warnings)"
elif static_test_evidence:
details = escape_markup(static_test_evidence)
elif s.checks_performed > 0:
details = f"{s.checks_performed} checks passed"
else:
details = "OK"
else:
parts = []
if s.errors > 0:
parts.append(f"{s.errors} errors")
if s.warnings > 0:
parts.append(f"{s.warnings} warnings")
details = ", ".join(parts) if parts else "Failed"
agent_eval = result.metadata.get("agent_eval")
if (
result.validator_name == "AGENT_EVAL"
and isinstance(agent_eval, dict)
and agent_eval.get("verdict") == "fail"
):
raw_summary = agent_eval.get("summary")
summary = raw_summary if isinstance(raw_summary, dict) else agent_eval
reason = agent_eval.get("verdict_reason") or summary.get("verdict_reason")
if isinstance(reason, str) and reason:
details = escape_markup(reason)
lift = _agent_eval_finite_number(summary.get("overall_lift"))
if lift is None:
lift = _agent_eval_finite_number(agent_eval.get("composite_lift"))
if lift is not None:
details += f"; lift [{lift_style(lift)}]{lift:+.2f}[/]"
if outcome in {"pass", "passed"}:
quality = result.metadata.get("quality_scores")
if isinstance(quality, dict):
score = _agent_eval_finite_number(quality.get("overall_score"))
if score is not None:
grade = str(quality.get("grade", "?"))
details += f"; quality {escape_markup(grade)} {score:.1f}/100"
agent = result.metadata.get("agent_eval")
if isinstance(agent, dict):
raw_summary = agent.get("summary")
summary = raw_summary if isinstance(raw_summary, dict) else agent
lift = _agent_eval_finite_number(summary.get("overall_lift", agent.get("composite_lift")))
if (
lift is not None
and agent.get("verdict") != "inconclusive"
and summary.get("verdict") != "inconclusive"
):
details += f"; lift [{lift_style(lift)}]{lift:+.2f}[/]"
elif agent.get("verdict") == "inconclusive":
details += "; inconclusive (skill never activated)"
table.add_row(escape_markup(str(result.validator_name)), status, details)
console.print(table)
for result in results:
agent_eval = result.metadata.get("agent_eval")
if isinstance(agent_eval, dict) and (comparison := agent_eval.get("prior_version_comparison")):
for line in version_comparison_lines(comparison):
console.print(escape_markup(line))
@staticmethod
def _static_test_evidence_message(result: ValidationResult) -> str | None:
"""Return the static test limitation from direct or folder-aggregated results."""
for detail in result.success_details:
if detail.check_name == "test_discovery":
return detail.message
for detail in result.success_details:
checks = detail.metadata.get("checks") if isinstance(detail.metadata, dict) else None
if not isinstance(checks, list):
continue
for check in checks:
if isinstance(check, dict) and check.get("name") == "test_discovery":
return "Target tests were not executed and coverage was not measured for any discovered skill"
return None
@staticmethod
def _is_advisory_agent_eval_skip(result: ValidationResult) -> bool:
"""Return whether an AGENT_EVAL result records a skipped live run."""
return is_advisory_agent_eval_skip(result)