Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

10.7 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Tests for skillevaluator.models.result.

Focuses on severity tallying, error/warning classification, and the
``recalculate_from_findings`` path used by the LLM false-positive downgrade
flow (validators/security.py, validators/policy.py).
"""

import pytest

from skillevaluator.models.result import (
    Finding,
    Severity,
    ValidationResult,
    ValidationSummary,
)


def _finding(severity: Severity, *, category: str = "TEST", check: str = "check") -> Finding:
    return Finding(
        category=category,
        severity=severity,
        check_name=check,
        message="something happened",
        file_path="skill/SKILL.md",
        line_number=3,
    )


class TestSeverity:
    """Tests for the Severity enum helpers."""

    @pytest.mark.parametrize(
        ("severity", "expected"),
        [
            (Severity.CRITICAL, True),
            (Severity.HIGH, True),
            (Severity.MEDIUM, False),
            (Severity.LOW, False),
            (Severity.INFO, False),
        ],
    )
    def test_is_error_default(self, severity: Severity, expected: bool):
        assert severity.is_error() is expected

    def test_is_error_fail_on_medium_and_low(self):
        assert Severity.MEDIUM.is_error(fail_on_medium=True) is True
        assert Severity.LOW.is_error(fail_on_low=True) is True
        assert Severity.LOW.is_error(fail_on_medium=True) is False


class TestFindingPostInit:
    """Finding normalizes string severities to the Severity enum."""

    def test_string_severity_is_coerced(self):
        finding = Finding(category="C", severity="high", check_name="c", message="m", file_path="f")
        assert finding.severity is Severity.HIGH

    def test_unknown_string_severity_falls_back_to_medium(self):
        finding = Finding(category="C", severity="bogus", check_name="c", message="m", file_path="f")
        assert finding.severity is Severity.MEDIUM


class TestValidationSummaryTally:
    """Direct tests for the tally / reset helpers."""

    def test_tally_increments_matching_counter(self):
        summary = ValidationSummary()
        summary.tally_severity(Severity.CRITICAL)
        summary.tally_severity(Severity.CRITICAL)
        summary.tally_severity(Severity.LOW)
        assert summary.critical_count == 2
        assert summary.low_count == 1
        assert summary.high_count == 0

    def test_info_and_non_enum_are_noops(self):
        summary = ValidationSummary()
        summary.tally_severity(Severity.INFO)
        summary.tally_severity("critical")  # raw string, not a Severity instance
        assert (summary.critical_count, summary.high_count, summary.medium_count, summary.low_count) == (0, 0, 0, 0)

    def test_reset_finding_counts_zeros_tracked_fields(self):
        summary = ValidationSummary(
            files_scanned=4,
            checks_performed=9,
            errors=2,
            warnings=1,
            critical_count=1,
            high_count=1,
            medium_count=3,
            low_count=2,
        )
        summary.reset_finding_counts()
        assert (summary.errors, summary.warnings) == (0, 0)
        assert (summary.critical_count, summary.high_count, summary.medium_count, summary.low_count) == (0, 0, 0, 0)
        # Untracked aggregate fields are preserved.
        assert summary.files_scanned == 4
        assert summary.checks_performed == 9


class TestAddFinding:
    """Tests for ValidationResult.add_finding (new + legacy calling styles)."""

    def test_high_finding_fails_and_counts(self):
        result = ValidationResult()
        result.add_finding(_finding(Severity.HIGH))
        assert result.passed is False
        assert result.summary.high_count == 1
        assert result.summary.errors == 1
        assert len(result.errors) == 1
        assert result.warnings == []

    def test_medium_finding_warns_by_default(self):
        result = ValidationResult()
        result.add_finding(_finding(Severity.MEDIUM))
        assert result.passed is True
        assert result.summary.medium_count == 1
        assert result.summary.warnings == 1
        assert len(result.warnings) == 1

    def test_medium_finding_fails_when_fail_on_medium(self):
        result = ValidationResult()
        result.add_finding(_finding(Severity.MEDIUM), fail_on_medium=True)
        assert result.passed is False
        assert result.summary.errors == 1

    def test_info_finding_is_not_severity_counted(self):
        result = ValidationResult()
        result.add_finding(_finding(Severity.INFO))
        assert result.passed is True
        assert (result.summary.critical_count, result.summary.high_count) == (0, 0)
        assert result.summary.medium_count == 0
        assert result.summary.low_count == 0
        assert result.summary.warnings == 1

    def test_legacy_positional_style(self):
        result = ValidationResult()
        result.add_finding("CVE", Severity.HIGH, "vulnerable dependency")
        assert result.passed is False
        assert result.errors == ["[CVE-HIGH] vulnerable dependency"]
        # Legacy string style does not populate structured findings.
        assert result.findings == []

    def test_legacy_style_requires_severity_and_message(self):
        result = ValidationResult()
        with pytest.raises(ValueError, match="Legacy add_finding"):
            result.add_finding("CVE")


class TestAddStructuredFinding:
    """Tests for add_structured_finding error/warning routing."""

    def test_error_marks_failed(self):
        result = ValidationResult()
        result.add_structured_finding(_finding(Severity.CRITICAL), is_error=True)
        assert result.passed is False
        assert result.summary.critical_count == 1
        assert len(result.errors) == 1

    def test_warning_does_not_fail(self):
        result = ValidationResult()
        result.add_structured_finding(_finding(Severity.LOW), is_error=False)
        assert result.passed is True
        assert result.summary.low_count == 1
        assert len(result.warnings) == 1


class TestRecalculateFromFindings:
    """Tests mirroring the LLM false-positive downgrade flow."""

    def test_downgrade_to_info_clears_error_state(self):
        result = ValidationResult()
        result.add_finding(_finding(Severity.HIGH))
        result.add_finding(_finding(Severity.CRITICAL))
        assert result.passed is False

        # Simulate the security validator downgrading both findings to INFO.
        for finding in result.findings:
            finding.severity = Severity.INFO
        result.recalculate_from_findings()

        assert result.passed is True
        assert result.errors == []
        assert result.summary.errors == 0
        assert (result.summary.critical_count, result.summary.high_count) == (0, 0)
        # Both findings now warn (INFO is not an error) but are not severity-counted.
        assert result.summary.warnings == 2

    def test_recalculate_is_idempotent(self):
        result = ValidationResult()
        result.add_finding(_finding(Severity.MEDIUM))
        result.add_finding(_finding(Severity.HIGH))

        before = result.summary.to_dict()
        result.recalculate_from_findings()
        after = result.summary.to_dict()

        # files_scanned / checks_performed are not rebuilt, so compare the rest.
        for key in ("errors", "warnings", "critical_count", "high_count", "medium_count", "low_count"):
            assert before[key] == after[key]


class TestRecalculateKeepingIndependent:
    """The rebuild that a severity change needs, with the lines no finding repeats put back."""

    @staticmethod
    def _result(*, incomplete: bool) -> ValidationResult:
        result = ValidationResult()
        result.add_finding(_finding(Severity.HIGH))
        result.add_error("scanner report was cut short")
        result.fail_gate("scanner status is unknown")
        result.add_warning("scanner skipped one file")
        result.add_warning("scanner skipped one file")
        if incomplete:
            result.mark_scan_incomplete("scanner")
        return result

    @staticmethod
    def _lower(result: ValidationResult, **options: bool) -> tuple[list[str], list[str]]:
        errors, warnings = result.independent_errors(), result.independent_warnings()
        result.findings[0].severity = Severity.INFO
        result.recalculate_keeping_independent(errors, warnings, **options)
        return errors, warnings

    @pytest.mark.parametrize("incomplete", [True, False])
    def test_errors_stay_only_for_an_incomplete_scan_and_a_gate_failure_stays_once(self, incomplete: bool):
        result = self._result(incomplete=incomplete)

        self._lower(result)

        expected = ["scanner status is unknown"]
        if incomplete:
            expected.append("scanner report was cut short")
        assert result.errors == expected
        assert result.summary.errors == len(expected)
        assert result.passed is False

    def test_every_error_stays_when_asked(self):
        result = self._result(incomplete=False)

        self._lower(result, keep_all_errors=True)

        assert result.errors == ["scanner status is unknown", "scanner report was cut short"]
        assert result.summary.errors == 2

    @pytest.mark.parametrize(("repeated", "copies"), [(False, 1), (True, 2)])
    def test_a_repeated_warning_goes_back_once_unless_asked(self, repeated: bool, copies: int):
        result = self._result(incomplete=False)

        self._lower(result, keep_repeated_warnings=repeated)

        assert result.warnings == [result.findings[0].to_legacy_string(), *["scanner skipped one file"] * copies]
        assert result.summary.warnings == 1 + copies

    def test_a_lowered_finding_is_counted_by_its_new_severity(self):
        result = self._result(incomplete=False)

        self._lower(result)

        assert (result.summary.high_count, result.summary.warnings) == (0, 2)
        assert result.findings[0].to_legacy_string() in result.warnings


class TestMerge:
    """Tests for merge and merge_with_prefix aggregation."""

    def test_merge_combines_findings_and_counts(self):
        a = ValidationResult()
        a.add_finding(_finding(Severity.MEDIUM))
        b = ValidationResult()
        b.add_finding(_finding(Severity.HIGH))

        a.merge(b)
        assert a.passed is False
        assert len(a.findings) == 2
        assert a.summary.medium_count == 1
        assert a.summary.high_count == 1

    def test_merge_with_prefix_tags_messages(self):
        parent = ValidationResult()
        child = ValidationResult()
        child.add_finding(_finding(Severity.HIGH))

        parent.merge_with_prefix(child, "my-skill")
        assert parent.passed is False
        assert all("[my-skill]" in err for err in parent.errors)
        assert parent.findings[0].file_path.startswith("[my-skill] ")