Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

6.2 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""PII findings are grouped by matched value, not repeated per line.

A doc-heavy skill repeating the same placeholder address produced 76
near-identical PII errors for 22 unique addresses; the report must show
one finding per (check, matched value) with an occurrence count and the
line list, so repeated values read as one row instead of pages of noise.
"""

from pathlib import Path

import pytest

from skillevaluator.validators.security import SecurityValidator


@pytest.fixture
def skill_with_repeated_emails(tmp_path) -> Path:
    skill = tmp_path / "mail-skill"
    skill.mkdir()
    (skill / "SKILL.md").write_text("---\nname: mail-skill\ndescription: mail helper\n---\n")
    workflows = skill / "workflows"
    workflows.mkdir()
    (workflows / "guide.md").write_text(
        "Intro line without addresses.\n"
        "Contact [email protected] for approval.\n"
        "Escalate to [email protected] when urgent.\n"
        "CC [email protected] on the thread.\n"
        "Archive mail from [email protected] monthly.\n"
    )
    return skill


def _email_findings(result):
    return [f for f in result.findings if f.check_name == "emails"]


class TestPiiGroupingByValue:
    @pytest.mark.parametrize("as_file", [True, False])
    def test_mdc_rule_and_workflow_pii_is_scanned(self, tmp_path, as_file):
        rule = tmp_path / "instructions.mdc"
        rule.write_text("---\ndescription: Contact rule\n---\nContact [email protected].\n", encoding="utf-8")
        target = rule if as_file else tmp_path
        result = SecurityValidator().validate_pii_only(target)
        assert "[email protected]" in {f.metadata.get("matched_value") for f in _email_findings(result)}

    def test_clean_mdc_has_no_pii(self, tmp_path):
        rule = tmp_path / "clean.mdc"
        rule.write_text("Respond briefly and use the configured contact directory.\n", encoding="utf-8")
        assert SecurityValidator().validate_pii_only(rule).passed

    def test_mdc_headings_and_slash_lines_are_scanned_like_markdown(self, tmp_path):
        rule = tmp_path / "escalation.mdc"
        rule.write_text(
            "---\ndescription: Escalation rule\n---\n# Escalate to [email protected]\n// Owner: [email protected]\n",
            encoding="utf-8",
        )
        found = {f.metadata.get("matched_value") for f in _email_findings(SecurityValidator().validate_pii_only(rule))}
        assert {"[email protected]", "[email protected]"} <= found

    def test_repeated_value_collapses_to_one_finding(self, skill_with_repeated_emails):
        result = SecurityValidator().validate_pii_only(skill_with_repeated_emails)

        findings = _email_findings(result)
        by_value = {f.metadata.get("matched_value"): f for f in findings}
        assert set(by_value) == {"[email protected]", "[email protected]"}

        boss = by_value["[email protected]"]
        assert boss.metadata["occurrence_count"] == 3
        assert [o["line"] for o in boss.metadata["occurrences"]] == [2, 3, 5]
        assert "[email protected]" in boss.message
        assert "3 occurrences" in boss.message
        assert "workflows/guide.md" in boss.message
        assert boss.line_number == 2

    def test_single_occurrence_message_names_the_value(self, skill_with_repeated_emails):
        result = SecurityValidator().validate_pii_only(skill_with_repeated_emails)

        alice = {f.metadata.get("matched_value"): f for f in _email_findings(result)}["[email protected]"]
        assert alice.metadata["occurrence_count"] == 1
        assert "[email protected]" in alice.message
        assert "occurrences" not in alice.message

    def test_error_count_matches_unique_values(self, skill_with_repeated_emails):
        result = SecurityValidator().validate_pii_only(skill_with_repeated_emails)

        email_errors = [e for e in result.errors if "email" in e.lower()]
        assert len(email_errors) == 2

    def test_groups_span_files(self, skill_with_repeated_emails):
        (skill_with_repeated_emails / "workflows" / "extra.md").write_text("Also ping [email protected] here.\n")

        result = SecurityValidator().validate_pii_only(skill_with_repeated_emails)

        boss = {f.metadata.get("matched_value"): f for f in _email_findings(result)}["[email protected]"]
        assert boss.metadata["occurrence_count"] == 4
        files = {o["file"] for o in boss.metadata["occurrences"]}
        assert files == {"workflows/guide.md", "workflows/extra.md"}
        assert "workflows/extra.md" in boss.message

    def test_distinct_values_stay_separate(self, tmp_path):
        skill = tmp_path / "two-skill"
        skill.mkdir()
        (skill / "SKILL.md").write_text("---\nname: two-skill\ndescription: two addresses\n---\n")
        (skill / "notes.md").write_text("Ping [email protected] then [email protected].\n")

        result = SecurityValidator().validate_pii_only(skill)

        values = {f.metadata.get("matched_value") for f in _email_findings(result)}
        # regex reports the first match per line; a second value on another line groups separately
        (skill / "notes2.md").write_text("Direct line to [email protected].\n")
        result = SecurityValidator().validate_pii_only(skill)
        values = {f.metadata.get("matched_value") for f in _email_findings(result)}
        assert values == {"[email protected]", "[email protected]"}


def test_folder_run_counts_folded_locations_once(tmp_path):
    from skillevaluator.models.result import ValidationResult
    from skillevaluator.reporting.finding_summary import summarize_findings

    skill = tmp_path / "support"
    skill.mkdir()
    (skill / "SKILL.md").write_text(
        "---\nname: support\ndescription: support helper\n---\nMail [email protected] now.\nCC [email protected] too.\n"
    )
    single = SecurityValidator(use_llm=False).validate_pii_only(skill)
    parent = ValidationResult()
    parent.merge_with_prefix(single, "support")

    [finding] = [f for f in parent.findings if f.check_name == "emails"]
    assert {item["file"] for item in finding.metadata["occurrences"]} == {finding.file_path}
    summary = summarize_findings([parent])
    assert summary["distinct_occurrences"] == summarize_findings([single])["distinct_occurrences"]