Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Prove the base install runs Tier 1 without Harbor or LLM extras.
A subprocess installs an import blocker for every extras-only top-level module,
then imports the CLI and runs static validation. This isolates from any heavy
modules other tests may have already imported into the parent interpreter.
"""
from __future__ import annotations
import os
import subprocess
import sys
from pathlib import Path
REPO = Path(__file__).resolve().parents[1]
FIXTURE = REPO / "tests" / "fixtures" / "skills" / "simple"
_SUBPROCESS = r"""
import os
import shutil
import sys
import tempfile
from pathlib import Path
BLOCKED = {
"anthropic", "boto3", "botocore", "harbor", "openai", "litellm",
"langgraph", "langgraph_checkpoint", "opentelemetry",
}
class _Blocker:
def find_spec(self, name, path=None, target=None):
if name.split(".")[0] in BLOCKED:
raise ModuleNotFoundError(f"blocked extras-only module: {name}")
return None
sys.meta_path.insert(0, _Blocker())
from click.testing import CliRunner
from skillevaluator.cli import cli
help_result = CliRunner().invoke(cli, ["--help"])
assert help_result.exit_code == 0, help_result.output
for name in (
"SKILL_EVAL_LLM_PROVIDER", "SKILL_EVAL_LLM_MODEL",
"SKILL_EVAL_LLM_API_KEY", "SKILL_EVAL_LLM_BASE_URL",
"NVIDIA_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY",
):
os.environ.pop(name, None)
models_result = CliRunner().invoke(cli, ["models"])
assert models_result.exit_code == 1, models_result.output
assert "No provider is configured" in models_result.output, models_result.output
with tempfile.TemporaryDirectory() as temporary_directory:
temporary_root = Path(temporary_directory)
skill = temporary_root / "simple"
shutil.copytree(FIXTURE, skill)
private_state = temporary_root / "private-state"
private_state.mkdir(mode=0o700)
os.environ["SKILLEVALUATOR_OUTPUT_PROVENANCE_KEY_FILE"] = str(private_state / "key")
previous_directory = Path.cwd()
os.chdir(skill)
try:
validate_result = CliRunner().invoke(
cli,
[
"validate",
".",
"--no-llm",
"--tiers",
"1",
"--checks",
"schema,quality,lint",
"--report",
"json",
],
)
finally:
os.chdir(previous_directory)
assert validate_result.exit_code == 0, validate_result.output
assert not (skill / "reports").exists()
assert (temporary_root / "simple-reports" / "BENCHMARK.md").is_file()
for tier in ("tier1", "tier2", "tier3"):
help_result = CliRunner().invoke(cli, [tier, "--help"])
assert help_result.exit_code == 0, help_result.output
direct_result = CliRunner().invoke(
cli, ["tier1", FIXTURE, "--no-llm", "--checks", "schema,quality,lint", "--report", "cli"]
)
assert direct_result.exit_code == 0, direct_result.output
tier1_doctor = CliRunner().invoke(cli, ["doctor", "--tier", "1", "--checks", "schema,quality,lint"])
assert tier1_doctor.exit_code == 0, tier1_doctor.output
tier2_doctor = CliRunner().invoke(cli, ["doctor", "--tier", "2", "--verify-models"])
assert tier2_doctor.exit_code == 1, tier2_doctor.output
assert "Tier 2 extra" in tier2_doctor.output, tier2_doctor.output
leaked = sorted(m for m in BLOCKED if m in sys.modules)
assert not leaked, f"base path imported extras-only modules: {leaked}"
print("BASE_OK")
"""
def test_base_install_runs_tier1_without_extras() -> None:
code = _SUBPROCESS.replace("FIXTURE", repr(str(FIXTURE)))
env = dict(os.environ, PYTHONPATH=str(REPO / "src"))
proc = subprocess.run(
[sys.executable, "-c", code],
capture_output=True,
text=True,
env=env,
)
assert proc.returncode == 0, f"stdout:\n{proc.stdout}\nstderr:\n{proc.stderr}"
assert "BASE_OK" in proc.stdout