Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Base validator class and result model.
This module provides the foundation for all SkillEvaluator validators with:
- Finding: Structured validation finding with full context for CI reporting
- ValidationResult: Dataclass for collecting validation outcomes
- ValidatorBase: Abstract base class with folder-level validation support
Note: The Finding and ValidationResult classes are now defined in skillevaluator.models.result
and re-exported here for backward compatibility. New code should import from
skillevaluator.models directly.
"""
from __future__ import annotations
import contextlib
import contextvars
import os
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterable, Iterator
from pathlib import Path
from skillevaluator.constants import SCAN_EXCLUDED_DIRS, SCAN_EXCLUDED_FILES, SKILL_MANIFEST_VARIANTS
# Import from the canonical location in models
from skillevaluator.models.result import (
Finding,
Severity,
SuccessDetail,
ValidationResult,
ValidationSummary,
)
from skillevaluator.utils.path_security import matches_filesystem_name
# Re-export for backward compatibility
__all__ = [
"Finding",
"Severity",
"SuccessDetail",
"ValidationResult",
"ValidationSummary",
"ValidatorBase",
"continue_on_failure_scope",
"iter_scannable_files",
]
# When enabled (via ``run_validation`` honoring ``--continue-on-failure``),
# batch folder validation records every skill instead of stopping at the first
# CRITICAL finding. Defaults to SkillEvaluator's historical stop-on-critical behavior.
_CONTINUE_ON_FAILURE: contextvars.ContextVar[bool] = contextvars.ContextVar(
"skillevaluator_continue_on_failure", default=False
)
@contextlib.contextmanager
def continue_on_failure_scope(enabled: bool) -> Iterator[None]:
"""Toggle continue-on-failure behavior for the duration of the scope.
While *enabled* is True, :meth:`ValidatorBase._validate_folder_or_skill`
keeps scanning and recording every skill in a folder even after a CRITICAL
finding, rather than returning early (parity with SkillEvaluator
``--continue-on-failure``).
"""
token = _CONTINUE_ON_FAILURE.set(enabled)
try:
yield
finally:
_CONTINUE_ON_FAILURE.reset(token)
def iter_scannable_files(
root: Path,
extensions: Iterable[str],
excluded_dirs: Iterable[str] | None = None,
excluded_files: Iterable[str] | None = None,
) -> list[Path]:
"""Walk ``root`` and return files matching any of ``extensions``.
Skips files whose path traverses any directory listed in ``excluded_dirs``
(defaults to :data:`SCAN_EXCLUDED_DIRS`). The exclusion matches at any
depth so that ``evals/results/.../foo.txt`` and a re-occurrence under
``references/evals/foo.md`` are both filtered out. Excluded directories
are pruned during the walk (never descended into), and symlinked
directories are not followed.
Skips generated publishing artifacts listed in ``excluded_files`` (defaults
to :data:`SCAN_EXCLUDED_FILES`) by basename, case-insensitively.
If ``root`` is itself a file, returns ``[root]`` when its suffix matches
one of ``extensions`` and its basename is not excluded (no directory walk
performed).
Args:
root: Directory or file path to scan.
extensions: Iterable of file extensions including the leading dot
(e.g. ``{".md", ".py"}``). Matching is case-sensitive on
case-sensitive filesystems (Linux), preserving the prior
``rglob("*.md")`` behavior of every Tier 1 call site this
helper replaced.
excluded_dirs: Directory names to skip at any depth. Pass an empty
iterable to disable exclusion entirely (e.g. for diagnostic
scans). ``None`` uses :data:`SCAN_EXCLUDED_DIRS`.
excluded_files: File basenames to skip. Pass an empty iterable to
disable filename exclusion. ``None`` uses
:data:`SCAN_EXCLUDED_FILES`.
Returns:
List of file paths in arbitrary order (callers that need a stable
order should sort the result themselves).
"""
excluded = SCAN_EXCLUDED_DIRS if excluded_dirs is None else frozenset(excluded_dirs)
excluded_basenames = SCAN_EXCLUDED_FILES if excluded_files is None else frozenset(excluded_files)
ext_set = set(extensions)
if root.is_file():
if matches_filesystem_name(root, excluded_basenames):
return []
return [root] if root.suffix in ext_set else []
# Prune excluded dirs during traversal instead of filtering rglob
# output: a skill that has been through Tier 3 can hold tens of
# thousands of artifact files under evals/results/, and an
# enumerate-then-filter walk still descends into all of them —
# once per extension. One pruned walk covers every extension.
suffixes = tuple(ext_set)
out: list[Path] = []
for dirpath, dirnames, filenames in os.walk(root):
base = Path(dirpath)
dirnames[:] = [d for d in dirnames if not matches_filesystem_name(base / d, excluded)]
for name in filenames:
if not name.endswith(suffixes):
continue
if matches_filesystem_name(base / name, excluded_basenames):
continue
out.append(Path(dirpath) / name)
return out
class ValidatorBase(ABC):
"""Abstract base class for skill validators.
Provides common functionality for:
- Finding SKILL.md files in directories
- Folder-level validation (validating multiple skills at once)
- Template method for consistent validation patterns
"""
# CRITICAL checks that judge one skill's own content, not the run's
# configuration: they fail that skill but never stop a folder scan early.
_FAIL_FAST_EXEMPT_CHECKS: frozenset[str] = frozenset()
@property
@abstractmethod
def name(self) -> str:
"""Return the validator name for display purposes."""
...
@property
@abstractmethod
def description(self) -> str:
"""Return a brief description of what this validator checks."""
...
@abstractmethod
def validate(self, skill_path: Path) -> ValidationResult:
"""Validate a skill or folder of skills.
Args:
skill_path: Path to skill directory or folder containing skills
Returns:
ValidationResult with pass/fail status and details
"""
...
def _find_exact_child(self, directory: Path, filename: str) -> Path | None:
"""Return a child path whose on-disk basename exactly matches ``filename``."""
try:
for child in directory.iterdir():
if child.is_file() and child.name == filename:
return child
except OSError:
return None
return None
def _find_skill_manifest(self, skill_path: Path) -> Path | None:
"""Locate the skill manifest, preferring canonical exact casing."""
for name in SKILL_MANIFEST_VARIANTS:
if manifest := self._find_exact_child(skill_path, name):
return manifest
return None
def _is_lowercase_manifest(self, skill_path: Path) -> bool:
"""Check if the skill manifest uses non-canonical lowercase naming.
Per agentskills.io spec, the canonical name is SKILL.md (uppercase).
On case-sensitive filesystems (Linux), skill.md is a distinct file
that won't be recognized by spec-compliant tooling.
"""
canonical = self._find_exact_child(skill_path, "SKILL.md")
lowercase = self._find_exact_child(skill_path, "skill.md")
return lowercase is not None and canonical is None
def _find_all_skills(self, root_path: Path) -> list[Path]:
"""Recursively find all skill directories containing SKILL.md files.
Manifests beneath an excluded directory below ``root_path`` (e.g.
``evals/``, ``results/``, ``.versions/`` in :data:`SCAN_EXCLUDED_DIRS`)
are skipped so that evaluation snapshots and version archives do not
get scanned as live skills.
"""
skill_dirs: set[Path] = set()
for manifest_name in SKILL_MANIFEST_VARIANTS:
for skill_md in root_path.rglob(manifest_name):
if any(part in SCAN_EXCLUDED_DIRS for part in skill_md.relative_to(root_path).parts):
continue
skill_dirs.add(skill_md.parent)
return sorted(skill_dirs)
def _is_skill_directory(self, path: Path) -> bool:
"""Check if path is a skill directory (contains SKILL.md)."""
return self._find_skill_manifest(path) is not None
def _validate_folder_or_skill(
self,
skill_path: Path,
single_skill_validator: Callable[[Path], ValidationResult],
action_description: str,
no_skills_fallback: bool = True,
) -> ValidationResult:
"""Template method for folder-level or single-skill validation.
Eliminates duplicated folder detection logic across validators.
Args:
skill_path: Path to validate
single_skill_validator: Function to validate a single skill
action_description: Description for progress messages (e.g., "Scanning")
no_skills_fallback: If True, validate folder directly when no skills found
Returns:
Aggregated ValidationResult for all skills
"""
if self._is_skill_directory(skill_path):
return single_skill_validator(skill_path)
result = ValidationResult()
skill_dirs = self._find_all_skills(skill_path)
if not skill_dirs:
if no_skills_fallback:
result.add_warning(f"No skills found in {skill_path}. {action_description} folder directly.")
return single_skill_validator(skill_path)
result.add_error(f"No skills found in {skill_path}. Expected SKILL.md files in skill directories.")
return result
# Use structured success details
result.add_success(
check_name="skill_discovery",
message=f"{action_description} {len(skill_dirs)} skill(s)",
skill_count=len(skill_dirs),
)
for skill_dir in skill_dirs:
skill_result = single_skill_validator(skill_dir)
skill_name = skill_dir.name
if skill_result.passed:
# Collect detailed check information from the skill result
check_details = [
{
"name": detail.check_name,
"description": detail.message,
"metadata": detail.metadata,
}
for detail in skill_result.success_details
]
# A reviewed baseline suppressing findings must stay visible in
# reports that show only this one-line summary.
notes = [
detail.message
for detail in skill_result.success_details
if detail.check_name == "skillspector_baseline" or "suppressed_count" in detail.metadata
]
result.add_success(
check_name=skill_name,
message="All checks passed" + (f"; {'; '.join(notes)}" if notes else ""),
checks=check_details,
total_checks=len(check_details),
)
else:
result.merge_with_prefix(skill_result, skill_name)
# Fail fast on CRITICAL findings (e.g., missing API keys). These are
# typically configuration errors that affect all skills, so by
# default we stop early. --continue-on-failure suppresses this so
# every skill is still scanned and recorded.
critical_findings = [
f
for f in skill_result.findings
if f.severity == Severity.CRITICAL and f.check_name not in self._FAIL_FAST_EXEMPT_CHECKS
]
if critical_findings and not _CONTINUE_ON_FAILURE.get():
# Return early with just this critical finding
return result
return result