Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

8.3 KB
Raw Blame
# SPDX-License-Identifier: Apache-2.0
# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
from __future__ import annotations

import json
import re

from skillevaluator.models.result import Finding, Severity, ValidationResult
from skillevaluator.reporting.run_status import Status, resolve_run_status


def failure_and_missing():
    result = ValidationResult(validator_name="Security")
    result.add_finding(
        Finding(
            category="SECURITY",
            severity=Severity.HIGH,
            check_name="blocked",
            message="Unsafe declaration",
            file_path="manifest.json",
        )
    )
    result.mark_scan_incomplete("scanner")
    return result


def test_real_failure_outranks_missing_evidence_in_same_result():
    result = failure_and_missing()
    status = resolve_run_status([result])
    assert status.headline == Status.FAILED
    assert status.of(result) == Status.FAILED
    assert status.missing == ("scanner",)


def test_failure_outranks_missing_evidence_in_another_result():
    missing = ValidationResult(validator_name="Missing")
    missing.mark_scan_incomplete("missing-tool")
    failed = ValidationResult(validator_name="Failed")
    failed.add_error("failed check")
    assert resolve_run_status([missing, failed]).headline == Status.FAILED
    assert resolve_run_status([failed, missing]).headline == Status.FAILED


def test_advisory_missing_evidence_does_not_change_exit_gate():
    missing = ValidationResult(validator_name="Advisory")
    missing.mark_scan_incomplete("optional-tool")
    missing.metadata["gating"] = {"tier": 2, "blocking": False}
    status = resolve_run_status([missing])
    assert status.headline == Status.PASSED
    assert status.of(missing) == Status.INCOMPLETE
    assert status.advisory
    assert status.publication == Status.INCOMPLETE


def test_clean_skip_is_distinct_from_pass():
    skipped = ValidationResult(validator_name="Skipped", metadata={"skipped": True})
    status = resolve_run_status([skipped])
    assert status.of(skipped) == Status.SKIPPED
    assert status.tiers[1].status == Status.SKIPPED


def test_report_surfaces_share_failure_precedence():
    from skillevaluator.reporting.cli import CLIReporter
    from skillevaluator.reporting.html import HTMLReporter
    from skillevaluator.reporting.json_reporter import JSONReporter
    from skillevaluator.reporting.markdown import MarkdownReporter
    from skillevaluator.reporting.sarif_reporter import SARIFReporter

    result = failure_and_missing()
    data = json.loads(JSONReporter().render_all([result]))
    assert data["overall_status"] == "failed"
    assert data["results"][0]["status"] == "failed"
    terminal = re.sub(r"\x1b\[[0-9;]*m", "", CLIReporter().render_all([result]))
    assert "[FAIL]" in terminal
    assert "**Status:** ❌ FAILED" in MarkdownReporter().render_all([result])
    html = HTMLReporter().render_all([result])
    embedded = re.search(r'<script id="report-data" type="application/json">(.*?)</script>', html, re.DOTALL)
    assert embedded is not None
    html_data = json.loads(embedded.group(1))
    assert html_data["summary"]["status"] == "failed"
    assert html_data["results"][0]["status"] == "failed"
    sarif = json.loads(SARIFReporter().render_all([result]))
    assert sarif["runs"][0]["properties"]["status"] == "failed"


def test_inconclusive_tier3_cannot_hide_a_confirmed_security_failure():
    result = failure_and_missing()
    result.validator_name = "AGENT_EVAL"
    result.metadata["agent_eval"] = {"verdict": "inconclusive", "summary": {"verdict": "inconclusive"}}
    status = resolve_run_status([result])
    assert status.of(result) == Status.FAILED
    assert status.headline == Status.FAILED
    assert status.missing == ("scanner",)


def test_confirmed_legacy_failure_survives_separate_missing_evidence():
    result = ValidationResult(validator_name="Code Integrity & Hygiene")
    result.add_error("Banned dependency found in one audited member")
    result.metadata["confirmed_failure"] = True
    result.mark_scan_incomplete("another-member")
    status = resolve_run_status([result])
    assert status.headline == Status.FAILED
    assert status.of(result) == Status.FAILED


def test_confirmed_failure_survives_a_folder_merge():
    """A folder keeps a member's confirmed failure; a member's gate failure alone stays missing evidence."""
    confirmed = ValidationResult(validator_name="Security Scan")
    confirmed.fail_gate("hooks/hooks.json: these hooks cannot pass this check")
    confirmed.metadata["confirmed_failure"] = True
    confirmed.mark_scan_incomplete("scanner")
    incomplete = ValidationResult(validator_name="Security Scan")
    incomplete.fail_gate("scanner reports incomplete analysis")
    incomplete.mark_scan_incomplete("scanner")

    for members, expected in (((confirmed, incomplete), Status.FAILED), ((incomplete,), Status.INCOMPLETE)):
        folder = ValidationResult(validator_name="Security Scan")
        for index, member in enumerate(members):
            folder.merge_with_prefix(member, f"skill-{index}")
        status = resolve_run_status([folder])
        assert status.of(folder) == expected
        assert status.headline == expected
        assert status.missing == ("scanner",)


def test_missing_scanner_process_error_alone_remains_incomplete():
    result = ValidationResult(validator_name="Scanner")
    result.add_error("The scanner produced no trustworthy report")
    result.mark_scan_incomplete("scanner")
    assert resolve_run_status([result]).headline == Status.INCOMPLETE


def test_resolved_results_are_classified_once(monkeypatch):
    from skillevaluator.reporting import run_status

    classified = []
    classify = run_status.result_status

    def track(result):
        classified.append(result)
        return classify(result)

    monkeypatch.setattr(run_status, "result_status", track)
    result = ValidationResult(validator_name="Static")
    status = resolve_run_status([result])
    assert status.of(result) == Status.PASSED
    assert status.of(result) == Status.PASSED
    assert classified == [result]


def test_unknown_results_are_classified_from_their_current_state():
    status = resolve_run_status([])
    result = ValidationResult(validator_name="Unknown")
    assert status.of(result) == Status.PASSED
    result.add_error("Failed after the run status was resolved")
    assert status.of(result) == Status.FAILED


def test_incomplete_results_scan_blocking_findings_once(monkeypatch):
    from skillevaluator.reporting import run_status

    scanned = []
    blocking_finding = run_status._blocking_finding

    def track(result):
        scanned.append(result)
        return blocking_finding(result)

    monkeypatch.setattr(run_status, "_blocking_finding", track)
    result = ValidationResult(validator_name="Scanner")
    result.mark_scan_incomplete("scanner")
    assert run_status.result_status(result) == Status.INCOMPLETE
    assert scanned == [result]


def test_gate_status_does_not_repeat_publication_payload_analysis(monkeypatch):
    from types import SimpleNamespace

    from skillevaluator.reporting import run_status

    calls = []

    def assess(*_args, **_kwargs):
        calls.append(True)
        return SimpleNamespace(status="incomplete")

    monkeypatch.setattr(run_status, "assess_publication", assess)
    status = resolve_run_status([ValidationResult(validator_name="Static")])
    assert status.headline == Status.PASSED
    assert calls == []
    assert status.publication == Status.INCOMPLETE
    assert status.publication == Status.INCOMPLETE
    assert calls == [True]


def test_cli_and_markdown_distinguish_explicit_na_from_missing_scores():
    from skillevaluator.reporting.cli import CLIReporter
    from skillevaluator.reporting.markdown import MarkdownReporter

    result = ValidationResult(validator_name="AGENT_EVAL")
    result.metadata["agent_eval"] = {
        "evaluators": {
            "accuracy": {
                "with_skill": None,
                "baseline": None,
                "lift": None,
                "with_skill_status": "not_applicable",
                "baseline_status": "missing",
            }
        }
    }
    terminal = CLIReporter().render(result)
    assert "N/A" in terminal and "NO SCORE" in terminal
    markdown = MarkdownReporter().render_all([result])
    assert "| Accuracy | N/A | NO SCORE | N/A |" in markdown