Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""PII findings are grouped by matched value, not repeated per line.
A doc-heavy skill repeating the same placeholder address produced 76
near-identical PII errors for 22 unique addresses; the report must show
one finding per (check, matched value) with an occurrence count and the
line list, so repeated values read as one row instead of pages of noise.
"""
from pathlib import Path
import pytest
from skillevaluator.validators.security import SecurityValidator
@pytest.fixture
def skill_with_repeated_emails(tmp_path) -> Path:
skill = tmp_path / "mail-skill"
skill.mkdir()
(skill / "SKILL.md").write_text("---\nname: mail-skill\ndescription: mail helper\n---\n")
workflows = skill / "workflows"
workflows.mkdir()
(workflows / "guide.md").write_text(
"Intro line without addresses.\n"
"Contact [email protected] for approval.\n"
"Escalate to [email protected] when urgent.\n"
"CC [email protected] on the thread.\n"
"Archive mail from [email protected] monthly.\n"
)
return skill
def _email_findings(result):
return [f for f in result.findings if f.check_name == "emails"]
class TestPiiGroupingByValue:
@pytest.mark.parametrize("as_file", [True, False])
def test_mdc_rule_and_workflow_pii_is_scanned(self, tmp_path, as_file):
rule = tmp_path / "instructions.mdc"
rule.write_text("---\ndescription: Contact rule\n---\nContact [email protected].\n", encoding="utf-8")
target = rule if as_file else tmp_path
result = SecurityValidator().validate_pii_only(target)
assert "[email protected]" in {f.metadata.get("matched_value") for f in _email_findings(result)}
def test_clean_mdc_has_no_pii(self, tmp_path):
rule = tmp_path / "clean.mdc"
rule.write_text("Respond briefly and use the configured contact directory.\n", encoding="utf-8")
assert SecurityValidator().validate_pii_only(rule).passed
def test_mdc_headings_and_slash_lines_are_scanned_like_markdown(self, tmp_path):
rule = tmp_path / "escalation.mdc"
rule.write_text(
"---\ndescription: Escalation rule\n---\n# Escalate to [email protected]\n// Owner: [email protected]\n",
encoding="utf-8",
)
found = {f.metadata.get("matched_value") for f in _email_findings(SecurityValidator().validate_pii_only(rule))}
assert {"[email protected]", "[email protected]"} <= found
def test_repeated_value_collapses_to_one_finding(self, skill_with_repeated_emails):
result = SecurityValidator().validate_pii_only(skill_with_repeated_emails)
findings = _email_findings(result)
by_value = {f.metadata.get("matched_value"): f for f in findings}
assert set(by_value) == {"[email protected]", "[email protected]"}
boss = by_value["[email protected]"]
assert boss.metadata["occurrence_count"] == 3
assert [o["line"] for o in boss.metadata["occurrences"]] == [2, 3, 5]
assert "[email protected]" in boss.message
assert "3 occurrences" in boss.message
assert "workflows/guide.md" in boss.message
assert boss.line_number == 2
def test_single_occurrence_message_names_the_value(self, skill_with_repeated_emails):
result = SecurityValidator().validate_pii_only(skill_with_repeated_emails)
alice = {f.metadata.get("matched_value"): f for f in _email_findings(result)}["[email protected]"]
assert alice.metadata["occurrence_count"] == 1
assert "[email protected]" in alice.message
assert "occurrences" not in alice.message
def test_error_count_matches_unique_values(self, skill_with_repeated_emails):
result = SecurityValidator().validate_pii_only(skill_with_repeated_emails)
email_errors = [e for e in result.errors if "email" in e.lower()]
assert len(email_errors) == 2
def test_groups_span_files(self, skill_with_repeated_emails):
(skill_with_repeated_emails / "workflows" / "extra.md").write_text("Also ping [email protected] here.\n")
result = SecurityValidator().validate_pii_only(skill_with_repeated_emails)
boss = {f.metadata.get("matched_value"): f for f in _email_findings(result)}["[email protected]"]
assert boss.metadata["occurrence_count"] == 4
files = {o["file"] for o in boss.metadata["occurrences"]}
assert files == {"workflows/guide.md", "workflows/extra.md"}
assert "workflows/extra.md" in boss.message
def test_distinct_values_stay_separate(self, tmp_path):
skill = tmp_path / "two-skill"
skill.mkdir()
(skill / "SKILL.md").write_text("---\nname: two-skill\ndescription: two addresses\n---\n")
(skill / "notes.md").write_text("Ping [email protected] then [email protected].\n")
result = SecurityValidator().validate_pii_only(skill)
values = {f.metadata.get("matched_value") for f in _email_findings(result)}
# regex reports the first match per line; a second value on another line groups separately
(skill / "notes2.md").write_text("Direct line to [email protected].\n")
result = SecurityValidator().validate_pii_only(skill)
values = {f.metadata.get("matched_value") for f in _email_findings(result)}
assert values == {"[email protected]", "[email protected]"}
def test_folder_run_counts_folded_locations_once(tmp_path):
from skillevaluator.models.result import ValidationResult
from skillevaluator.reporting.finding_summary import summarize_findings
skill = tmp_path / "support"
skill.mkdir()
(skill / "SKILL.md").write_text(
"---\nname: support\ndescription: support helper\n---\nMail [email protected] now.\nCC [email protected] too.\n"
)
single = SecurityValidator(use_llm=False).validate_pii_only(skill)
parent = ValidationResult()
parent.merge_with_prefix(single, "support")
[finding] = [f for f in parent.findings if f.check_name == "emails"]
assert {item["file"] for item in finding.metadata["occurrences"]} == {finding.file_path}
summary = summarize_findings([parent])
assert summary["distinct_occurrences"] == summarize_findings([single])["distinct_occurrences"]