Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""External tool execution utilities.
Provides reusable patterns for running external CLI tools, parsing their
JSON output, and handling errors consistently.
"""
from __future__ import annotations
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import time
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from pathlib import Path
from skillevaluator.logging_config import get_logger
from skillevaluator.models.result import Severity
from skillevaluator.utils.scanner_install import (
GITLEAKS_VERSION,
SKILLSPECTOR_VERSION,
SKILLSPECTOR_WHEEL_URL,
ensure_gitleaks,
ensure_skillspector_wheel,
gitleaks_platform,
managed_gitleaks_path,
managed_skillspector_wheel,
)
logger = get_logger(__name__)
# CVSSv3 score thresholds for CVE severity mapping
CVSS_THRESHOLDS: dict[Severity, float] = {
Severity.CRITICAL: 9.0,
Severity.HIGH: 7.0,
Severity.MEDIUM: 4.0,
Severity.LOW: 0.1,
}
# Gitleaks normally uses exit 1 for both findings and operational errors.
# Every invocation overrides the findings code so those states are distinct.
GITLEAKS_FINDINGS_EXIT_CODE = 10
def cvss_to_severity(score: float) -> Severity:
"""Convert CVSSv3 score to severity level."""
if score >= CVSS_THRESHOLDS[Severity.CRITICAL]:
return Severity.CRITICAL
if score >= CVSS_THRESHOLDS[Severity.HIGH]:
return Severity.HIGH
if score >= CVSS_THRESHOLDS[Severity.MEDIUM]:
return Severity.MEDIUM
return Severity.LOW
def resolve_tool_path(command: str, *, prefer_interpreter_sibling: bool = False) -> str | None:
"""Resolve a tool binary from PATH and the interpreter's own bin dir.
Isolated installs (``uv tool install``, ``pipx``) place the console
scripts of bundled extras next to the interpreter, in a directory
that is not on the user's PATH. Bundled scanners prefer that sibling
location so an unrelated global executable cannot silently replace the
version installed with SkillEvaluator. External tools such as Gitleaks
retain normal PATH precedence.
"""
if prefer_interpreter_sibling:
if sys.executable:
sibling = shutil.which(command, path=str(Path(sys.executable).parent))
if sibling is not None:
return sibling
return shutil.which(command)
path = shutil.which(command)
if path is not None:
return path
if not sys.executable:
return None
return shutil.which(command, path=str(Path(sys.executable).parent))
def _packaged_uv() -> str | None:
"""Return the uv executable installed by SkillEvaluator's ``uv`` dependency.
A ``pip install --user`` or ``--target`` layout puts it in a scripts
directory that is neither on PATH nor next to the interpreter.
"""
try:
from uv import find_uv_bin
return find_uv_bin()
except (ImportError, OSError):
return None
@dataclass
class ToolResult:
"""Result from running an external tool.
``success`` only means the process launched and ran to completion
(no missing binary, timeout, or OS error) — NOT that the tool
reported a clean scan. Callers must branch on ``exit_code`` to
distinguish clean (usually 0) from findings (usually 1) from tool
errors (anything else); treating an unexpected exit code as a clean
result makes a crashed scanner indistinguishable from a passing one.
"""
success: bool
stdout: str
stderr: str
exit_code: int
error_message: str | None = None
def _captured_text(value: str | bytes | None) -> str:
"""Normalize partial subprocess output without dropping undecodable bytes."""
if value is None:
return ""
if isinstance(value, bytes):
return value.decode("utf-8", errors="replace")
return value
def _skillspector_version(stdout: str, stderr: str) -> str | None:
"""Read an unambiguous SkillSpector identity line, preserving release suffixes."""
pattern = r"skillspector(?:,?\s+version)?\s+v?(\d+\.\d+\.\d+[A-Za-z0-9._+-]*)"
versions = {
match.group(1)
for line in (stdout + "\n" + stderr).splitlines()
if (match := re.fullmatch(pattern, line.strip(), flags=re.IGNORECASE))
}
return versions.pop() if len(versions) == 1 else None
class ExternalTool:
"""Wrapper for external CLI tools with consistent error handling.
Provides automatic PATH lookup, installation hints, timeout handling,
and structured result objects.
"""
def __init__(
self,
name: str,
command: str | None = None,
*,
prefer_interpreter_sibling: bool = False,
override_env: str | None = None,
managed: bool = False,
):
"""Initialize tool wrapper.
Args:
name: Human-readable tool name for error messages
command: Command name to look up in PATH (defaults to name)
prefer_interpreter_sibling: Prefer the executable installed next
to SkillEvaluator's interpreter over a same-named PATH tool.
override_env: Optional environment variable containing an explicit
absolute executable path. Invalid overrides fail closed rather
than silently falling back to another executable.
managed: Fall back to SkillEvaluator's pinned provisioning when the
tool is missing, or automatically discovered SkillSpector is not
the pinned release: the verified SkillSpector wheel through
``uv tool run`` (a uv on PATH, else the one installed with
SkillEvaluator), or the downloaded Gitleaks binary.
"""
self.name = name
self.command = command or name
self.override_env = override_env
self._prefer_interpreter_sibling = prefer_interpreter_sibling
self._managed = managed
self.refresh()
def refresh(self) -> None:
"""Resolve the executable again, e.g. after a managed install."""
self.resolution_source = "automatic"
self._configuration_error: str | None = None
self._version: str | None = None
self._uv_run: tuple[str, tuple[str, ...]] | None = None
self._ensure: tuple[str, Callable[[], str | None]] | None = None
self._skillspector_candidate: str | None = None
override_env = self.override_env
name = self.name
override = os.environ.get(override_env, "").strip() if override_env else ""
if override:
candidate = Path(override)
if not candidate.is_absolute() or not candidate.is_file() or not os.access(candidate, os.X_OK):
self._path = None
self._configuration_error = f"{override_env} must name an absolute executable file; got {override!r}"
else:
self._path = str(candidate.resolve())
self.resolution_source = f"override:{override_env}"
logger.warning("Using explicit %s executable override from %s: %s", name, override_env, self._path)
else:
self._path = resolve_tool_path(
self.command,
prefer_interpreter_sibling=self._prefer_interpreter_sibling,
)
if self._managed:
if self._path is not None and self.command == "skillspector":
# Discovery must stay cheap and must never execute a PATH
# program during module import or unrelated CLI commands.
self._skillspector_candidate = self._path
elif self._path is None:
self._resolve_managed()
def _require_pinned_skillspector(self, version: str | None) -> None:
"""Keep automatic SkillSpector resolution on the supported pinned release."""
if version == SKILLSPECTOR_VERSION:
return
detail = (
f"Automatically discovered SkillSpector {version or '(version could not be verified)'} "
f"at {self._path}; expected pinned version {SKILLSPECTOR_VERSION}."
)
self._path = None
self._version = None
self._resolve_managed()
override = self.override_env or "SKILLEVALUATOR_SKILLSPECTOR_PATH"
if self._path is not None:
logger.warning(
"%s Using the managed runtime via uv tool run. Set %s to an absolute executable path "
"for an intentional override.",
detail,
override,
)
else:
self._configuration_error = (
f"{detail} The pinned runtime requires uv and a valid SKILLEVALUATOR_RUNTIME_DIR; "
f"install uv or correct the runtime directory, or set {override} to an absolute executable path "
"for an intentional override."
)
logger.warning("%s", self._configuration_error)
def _resolve_managed(self) -> None:
if self.command == "skillspector":
# A uv on PATH keeps precedence; the packaged one covers a plain pip install.
uv = resolve_tool_path("uv") or _packaged_uv()
wheel = managed_skillspector_wheel()
if uv is not None and wheel is not None:
self._path = uv
self._uv_run = (uv, ("tool", "run", "--quiet", "--from", str(wheel), "skillspector"))
self._ensure = (uv, ensure_skillspector_wheel)
self.resolution_source = "uv tool run"
elif self.command == "gitleaks":
binary = managed_gitleaks_path()
if binary is not None and gitleaks_platform() is not None:
self._path = str(binary)
self._ensure = (self._path, ensure_gitleaks)
self.resolution_source = "managed"
def _launch_args(self) -> tuple[str, ...]:
if self._uv_run is not None and self._path == self._uv_run[0]:
return self._uv_run[1]
return ()
@property
def is_available(self) -> bool:
"""Check if tool is available in PATH."""
return self._path is not None
@property
def path(self) -> str | None:
"""Get full path to tool binary."""
return self._path
@property
def version(self) -> str | None:
"""The pinned release for managed installs, otherwise what the tool's version command reports."""
if self._skillspector_candidate is not None and self._skillspector_candidate == self._path:
self.run(["--version"], timeout=30, log_command=False)
if self.resolution_source == "uv tool run":
return SKILLSPECTOR_VERSION
if self.resolution_source == "managed":
return GITLEAKS_VERSION
if self._path is None:
return None
if self._version is None:
argv = [self._path, "version" if self.command == "gitleaks" else "--version"]
options = {}
if self.command == "skillspector":
from skillevaluator.validators.security import _skillspector_process_env
options["env"] = _skillspector_process_env()
try:
proc = subprocess.run(
argv, capture_output=True, text=True, errors="replace", timeout=30, check=False, **options
)
except (OSError, subprocess.SubprocessError):
proc = None
if self.command == "skillspector":
self._version = (
_skillspector_version(proc.stdout or "", proc.stderr or "")
if proc is not None and proc.returncode == 0
else ""
) or ""
else:
match = proc and (
re.search(r"\d+\.\d+(?:\.\d+)?", proc.stdout or "")
or re.search(r"\d+\.\d+(?:\.\d+)?", proc.stderr or "")
)
self._version = match.group(0) if match else ""
return self._version or None
@property
def configuration_error(self) -> str | None:
"""Return why executable selection could not satisfy the scanner configuration."""
return self._configuration_error
def get_install_hint(self) -> str:
"""Get installation hint for this tool."""
if self._configuration_error:
return self._configuration_error
bundled_scanner_hint = (
"Bundled with SkillEvaluator — reinstall with: "
'uv tool install --reinstall "skillevaluator @ git+https://github.com/NVIDIA/SkillEvaluator.git"'
)
security_extra_hint = (
"Bundled with the security extra — reinstall with: "
'uv tool install "skillevaluator[all] @ git+https://github.com/NVIDIA/SkillEvaluator.git"'
)
gitleaks_go = "go install github.com/zricethezav/gitleaks/v8@latest"
if sys.platform == "win32":
gitleaks_hint = f"Install with: winget install Gitleaks.Gitleaks or {gitleaks_go}"
semgrep_hint = "Optional; install separately with: uv tool install semgrep"
elif sys.platform == "darwin":
gitleaks_hint = f"Install with: brew install gitleaks or {gitleaks_go}"
semgrep_hint = "Optional; install separately with: brew install semgrep or uv tool install semgrep"
else:
gitleaks_hint = (
f"Install with: {gitleaks_go}, or a release binary from https://github.com/gitleaks/gitleaks/releases"
)
semgrep_hint = "Optional; install separately with: uv tool install semgrep"
hints = {
"gitleaks": gitleaks_hint,
"pip-audit": security_extra_hint,
"safety": "Optional secondary scanner; install separately with: uv tool install safety",
"bandit": bundled_scanner_hint,
"semgrep": semgrep_hint,
"skillspector": f"Install with: uv tool install {SKILLSPECTOR_WHEEL_URL}",
"skillevaluator": (
'Install with: uv tool install "skillevaluator[all] @ git+https://github.com/NVIDIA/SkillEvaluator.git"'
),
}
return hints.get(self.command, f"Install {self.name} and ensure it's in PATH")
def run(
self,
args: list[str],
*,
cwd: Path | None = None,
timeout: int = 120,
log_command: bool = True,
env: Mapping[str, str] | None = None,
replace_env: bool = False,
) -> ToolResult:
"""Execute tool with given arguments.
Args:
args: Command arguments (tool path prepended automatically)
cwd: Working directory for command
timeout: Timeout in seconds
log_command: Whether to log the command being run
env: Extra environment variables merged over the parent
environment for this invocation only
replace_env: Use ``env`` as the complete child environment instead
of merging it over the parent. Use this for credential-isolated
subprocesses.
Returns:
ToolResult with stdout, stderr, and error info
"""
child_env = dict(env or {}) if replace_env else ({**os.environ, **env} if env else None)
requested_timeout = timeout
if self._skillspector_candidate is not None and self._skillspector_candidate == self._path:
from skillevaluator.validators.security import _skillspector_process_env
self._skillspector_candidate = None
started = time.monotonic()
try:
probe = subprocess.run(
[self._path, "--version"],
capture_output=True,
text=True,
errors="replace",
timeout=min(30, timeout),
check=False,
env=_skillspector_process_env(child_env),
)
except (OSError, subprocess.SubprocessError):
probe = None
version = (
_skillspector_version(probe.stdout or "", probe.stderr or "")
if probe is not None and probe.returncode == 0
else None
)
self._version = version or ""
self._require_pinned_skillspector(version)
if version == SKILLSPECTOR_VERSION and args == ["--version"]:
return ToolResult(True, probe.stdout or "", probe.stderr or "", probe.returncode)
timeout -= time.monotonic() - started
if not self.is_available:
return ToolResult(
success=False,
stdout="",
stderr="",
exit_code=-1,
error_message=self.configuration_error or f"{self.name} not installed. {self.get_install_hint()}",
)
if timeout <= 0:
return ToolResult(
False, "", "", -1, error_message=f"{self.name} timed out after {requested_timeout} seconds"
)
launch_args = self._launch_args()
if self._ensure is not None and self._path == self._ensure[0] and (error := self._ensure[1]()):
return ToolResult(success=False, stdout="", stderr="", exit_code=-1, error_message=error)
cmd = [self._path, *launch_args, *args]
if log_command:
logger.info(f"Running {self.name}: {' '.join(cmd[:5])}...")
try:
# Text mode decodes with Python's selected codec (the locale codec,
# or UTF-8 in UTF-8 mode), which is what Python scanners write. A
# strict decode of bytes outside that codec (a Windows traceback
# with a CJK path, for example) raises UnicodeDecodeError in the
# reader thread and leaves proc.stdout/stderr as None, so replace
# undecodable bytes instead of failing.
proc = subprocess.run(
cmd,
capture_output=True,
text=True,
errors="replace",
timeout=timeout,
cwd=cwd,
env=child_env,
)
return ToolResult(
success=True,
stdout=_captured_text(proc.stdout),
stderr=_captured_text(proc.stderr),
exit_code=proc.returncode,
)
except subprocess.TimeoutExpired as exc:
return ToolResult(
success=False,
stdout=_captured_text(exc.stdout),
stderr=_captured_text(exc.stderr),
exit_code=-1,
error_message=f"{self.name} timed out after {requested_timeout} seconds",
)
except FileNotFoundError:
return ToolResult(
success=False,
stdout="",
stderr="",
exit_code=-1,
error_message=f"{self.name} binary not found in PATH",
)
except OSError as e:
return ToolResult(
success=False,
stdout="",
stderr="",
exit_code=-1,
error_message=f"{self.name} execution failed: {e}",
)
def parse_json_output(output: str, on_error: str | None = None) -> dict | list | None:
"""Parse JSON output from external tool.
Args:
output: Raw stdout from tool
on_error: Optional message to check for in non-JSON output
Returns:
Parsed JSON data or None if parsing fails
"""
if not output or not output.strip():
return None
try:
return json.loads(output)
except json.JSONDecodeError:
if on_error and on_error in output:
return {}
return None
def create_temp_config(content: str, suffix: str = ".toml") -> Path:
"""Create a temporary configuration file for external tool configs.
Caller is responsible for cleanup; consider using contextlib for scoped lifetime.
Args:
content: Configuration file content
suffix: File extension (default: .toml for Gitleaks)
Returns:
Path to temporary file
"""
fd, path_str = tempfile.mkstemp(suffix=suffix, prefix="skillevaluator_")
os.close(fd)
config_path = Path(path_str)
config_path.write_text(content)
return config_path
class Tools:
"""Registry of external tools.
Provides pre-configured ExternalTool instances for security scanning
tools and evaluation tools used by SkillEvaluator validators.
"""
gitleaks = ExternalTool("Gitleaks", "gitleaks", managed=True)
pip_audit = ExternalTool("pip-audit", "pip-audit", prefer_interpreter_sibling=True)
safety = ExternalTool("Safety", "safety")
bandit = ExternalTool(
"Bandit",
"bandit",
prefer_interpreter_sibling=True,
override_env="SKILLEVALUATOR_BANDIT_PATH",
)
semgrep = ExternalTool(
"Semgrep",
"semgrep",
prefer_interpreter_sibling=True,
override_env="SKILLEVALUATOR_SEMGREP_PATH",
)
skillspector = ExternalTool(
"skillspector",
"skillspector",
prefer_interpreter_sibling=True,
override_env="SKILLEVALUATOR_SKILLSPECTOR_PATH",
managed=True,
)