Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
Show each serious finding of a failed Tier 1 check in what to fix (#621)
Christopher Kevinf83c2edcb65f
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex e62af359..9e9266a2 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -305,6 +305,9 @@ All notable changes to SkillEvaluator are documented in this file.(`CC-FIELD-VALUE`): an `effort` other than low, medium, high, xhigh, max or a number,a `shell` other than bash or powershell, `background` without `context: fork`, and`maxTurns`. The skill still loads.+- `validate`'s Tier 1 "what to fix" lists each critical or high finding of a failed+ check, not only the first: a security scan with several SkillSpector findings+ shows them one per line, up to three, then "+N more in detailed reports".- Copilot CLI `harness` results flag an empty or whitespace-only `description`(`CP-DESC-EMPTY`): Copilot lists the skill with a blank description, so themodel sees only its name.diff --git a/src/skillevaluator/reporting/console_ui.py b/src/skillevaluator/reporting/console_ui.pyindex 5bc40ea2..c35502b5 100644--- a/src/skillevaluator/reporting/console_ui.py+++ b/src/skillevaluator/reporting/console_ui.py@@ -1266,7 +1266,7 @@ def summarize_tier1(results: list[ValidationResult], lineup: list[str] | None =rows.append(TierRow("quality", [(grade, f"bold {grade_style(grade, status=status)}"), (f" {score:.1f} / 100", TEXT)]))- items = [_tier1_fix(r) for r in broken]+ items = [item for r in broken for item in _tier1_fixes(r)]if setup := [r for r in incomplete if not _only_llm_stages_missing(r)]:items.append(_setup_fix(setup))if llm := [r for r in incomplete if _only_llm_stages_missing(r)]:@@ -1274,6 +1274,23 @@ def summarize_tier1(results: list[ValidationResult], lineup: list[str] | None =return not failed, rows + fix_section_rows(items)+def _tier1_fixes(result: ValidationResult) -> list[FixItem]:+ """One item per distinct critical/high finding, so a check that failed on several shows each one."""+ serious = [+ f for f in result.findings if getattr(f.severity, "value", str(f.severity)).lower() in ("critical", "high")+ ]+ messages = list(dict.fromkeys(f.message for f in serious))+ if result.gate_failures or len(messages) < 2:+ return [_tier1_fix(result)]+ where = _short_name(result.validator_name)+ items, shown = [], set()+ for message in messages:+ action = next((f.suggestion for f in serious if f.message == message and f.suggestion), None)+ items.append(FixItem(where, message, [action] if action and action not in shown else []))+ shown.add(action)+ return items++def _tier1_fix(result: ValidationResult) -> FixItem:problem = _first_error(result) or f"{result.validator_name} failed"action = next((f.suggestion for f in result.findings if f.message == problem and f.suggestion), None)diff --git a/tests/test_commands.py b/tests/test_commands.pyindex 17cbb1f6..9a2d78f1 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -3372,6 +3372,42 @@ def test_render_evaluation_result_invokes_findings_report(monkeypatch) -> None:assert str(results_dir) == "/tmp/run"+def test_summarize_tier1_lists_each_serious_finding() -> None:+ from skillevaluator.models.result import Finding, Severity, ValidationResult+ from skillevaluator.reporting.console_ui import summarize_tier1++ failing = ValidationResult(validator_name="Security Scan", passed=False)+ for message, severity, suggestion in (+ ("Tool Misuse: rm -rf ./build", Severity.HIGH, "Reject dangerous parameters."),+ ("YARA Match: rm -rf ./; without asking", Severity.HIGH, "Reject dangerous parameters."),+ ("Hook bypass: git commit --no-verify", Severity.CRITICAL, "Let the hooks run."),+ ("Unpinned npx package", Severity.HIGH, "Pin the package version."),+ ("Excessive Agency: without asking", Severity.MEDIUM, "Ask before acting."),+ ):+ failing.add_finding(+ Finding(+ category="SECURITY",+ severity=severity,+ check_name="skillspector",+ message=message,+ file_path="SKILL.md",+ suggestion=suggestion,+ )+ )++ _ok, rows = summarize_tier1([failing])+ rendered = ["".join(chunk for chunk, _style in row.segments) for row in rows]+ numbered = [row for row in rows if row.glyph in {"1", "2", "3"}]+ assert [row.segments[0][0] for row in numbered] == [+ "Tool Misuse: rm -rf ./build",+ "YARA Match: rm -rf ./; without asking",+ "Hook bypass: git commit --no-verify",+ ]+ assert sum("Reject dangerous parameters." in line for line in rendered) == 1+ assert any("+1 more in detailed reports" in line for line in rendered)+ assert not any("Excessive Agency" in line for line in rendered)++def test_summarize_tier2_lists_duplicate_details() -> None:# "duplicates found" alone is useless — the section must say WHAT# duplicated, up to three findings plus an overflow pointer.