Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

8.9 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

[build-system]
build-backend = "setuptools.build_meta"
requires = ["setuptools >= 77", "wheel"]

[project]
name = "skillevaluator"
version = "1.0.1"
description = "Multi-tier framework for evaluating AI agent skills with quality gates, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation."
authors = [{ name = "NVIDIA Corporation" }]
readme = "README.md"
keywords = [
    "skills-evaluation",
    "skills",
    "skill-uplift",
    "ai-agents",
    "security",
    "vulnerability-scanner",
    "claude-code",
]
classifiers = [
    "Environment :: Console",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Programming Language :: Python :: 3.14",
    "Topic :: Software Development :: Quality Assurance",
]
requires-python = ">=3.12,<3.15"
license = "Apache-2.0"
license-files = [
    "LICENSE",
    "NOTICE",
    "THIRD_PARTY_NOTICES.md",
]
# Base install is static-validation capable (Tier 1) and must NOT pull Harbor
# or LLM clients. Those live behind the extras below.
dependencies = [
    # Tier 1 Code Risk Analysis scanner, which validate runs by default.
    "bandit>=1.9.4",
    "click>=8.3.3,<9",
    "idna>=3.10,<4",
    "jinja2>=3.1",
    "markdown-it-py>=4.2,<5",
    "packaging>=24",
    "pydantic>=2.11.0",
    "pyyaml>=6.0",
    # Dataset-authored signal patterns require a per-match timeout.
    "regex>=2024.11.6",
    "rich>=13.5.0",
    # Runs the pinned SkillSpector wheel in its own environment (uv tool run),
    # so a plain pip install can complete the Tier 1 security check.
    "uv>=0.11.27",
]

[project.optional-dependencies]
# Shared public LLM / embedding clients used by LLM-backed checks, Tier 2, and Tier 3.
llm = [
    "anthropic>=0.83.0",
    "boto3>=1.34.0",
    "litellm>=1.92.0,<1.94.0.dev0",
    "openai>=2.21.0",
]
# Tier 2 intra-skill deduplication and local-catalog inter-skill similarity.
tier2 = [
    "skillevaluator[llm]",
]
# Tier 3 live agent evaluation through Harbor's native environments.
tier3 = [
    "skillevaluator[llm]",
    "harbor[gke]==0.25.0",
    # CVE-2026-59950, CVE-2026-52870, and CVE-2026-52869 fixed floor.
    "mcp>=1.28.1,<2",
    # CVE-2026-48522 through CVE-2026-48526 fixed floor.
    "pyjwt[crypto]>=2.13.0",
    # GHSA-hj66-6f7g-4r5v and GHSA-xpv3-w29h-x7cv fixed floor (via harbor[gke]).
    "oauthlib>=4.0.0",
    # PYTHON_DOTENV_DISABLED, which keeps Harbor from loading .env files, needs 1.2.0.
    "python-dotenv>=1.2.0",
]
# Scanner for the opt-in Dependency Vulnerability Audit (`--check dependency`).
security = [
    "pip-audit>=2.10.0",
]
# Verify GitHub-signed execution receipts for the optional CI report publisher.
publisher = [
    "pyjwt[crypto]>=2.13.0",
]
# Optional local UI for comparing saved reports; never imported by base commands.
dashboard = [
    # server.allowedHosts, which limits the dashboard to loopback host names, needs 1.61.
    "streamlit>=1.61,<2",
]
# Full public install convenience bundle.
all = [
    "skillevaluator[tier2,tier3,security]",
]
dev = [
    "skillevaluator[publisher]",
    # PyPA build backs the `make build` target.
    "build>=1.2",
    "httpx>=0.27",
    "pytest>=8.4.0",
    "pytest-cov>=6.2.0",
    # pytest-xdist carried over for parallel test execution (-n auto).
    "pytest-xdist>=3.6.0",
    # Dedicated offline HTML browser checks; never needed by the installed reporter.
    "playwright==1.65.2",
    # Ruff 0.16 enables new rules that require a dedicated repository-wide cleanup.
    "ruff>=0.15.4,<0.16",
]

[project.urls]
Documentation = "https://docs.nvidia.com/skills/skillevaluator/"
Repository = "https://github.com/NVIDIA/SkillEvaluator"

[project.scripts]
# Single console script for the consolidated distribution.
skillevaluator = "skillevaluator.cli:cli"

[tool.uv]
managed = true

[tool.setuptools.packages.find]
where = ["src"]
include = ["skillevaluator*"]

[tool.setuptools.package-data]
"skillevaluator.tier3" = ["grader_reference/*.json", "grader_reference/trajectories/*.json"]
"skillevaluator" = ["ci-release.json", "ci-release-all-tiers.json"]
"skillevaluator.config" = ["*.yaml", "profiles/*.yaml"]
"skillevaluator.harness" = ["profiles/*.yaml"]
"skillevaluator.reporting" = ["templates/*.j2", "assets/*.js", "assets/*.md"]
"skillevaluator.tier3.harbor" = [
    "templates/*.py",
    "templates/*.sh",
]
"skillevaluator.tier3.reference_skills" = [
    "*/SKILL.md",
    "*/evals/*.json",
    "*/evals/*.jsonl",
    "*/evals/*.yaml",
    "*/evals/*.yml",
    "*/evals/harbor/**/*",
    "*/evals/files/**/*",
    "*/references/**/*",
    "*/scripts/*.py",
    "*/scripts/*.sh",
    "*/files/**/*",
    "*/environment/docker-compose*.yml",
    "*/environment/docker-compose*.yaml",
    "*/mcp_servers.toml",
]

[tool.ruff]
line-length = 120
target-version = "py312"
src = ["src", "tests"]
exclude = ["**/evals/results/**", ".venv", "build", "dist"]

[tool.ruff.lint]
# Union of the established rule selections and the broader static-analysis set.
select = ["E", "F", "I", "N", "W", "UP", "B", "A", "COM", "C4", "DTZ", "T10", "EXE", "FA", "ISC", "ICN", "G", "INP", "PIE", "T20", "Q", "RSE", "RET", "SLF", "SLOT", "SIM", "TID", "TCH", "INT", "ARG", "PTH", "TD", "FIX", "ERA", "PD", "PGH", "PL", "FLY", "NPY", "AIR", "PERF", "FURB", "LOG", "RUF"]
# Union of the ignore lists across all three repos.
ignore = [
    "E402",      # Module-level import not at top of file
    "E501",      # Line too long
    "E741",      # Ambiguous variable name
    "N806",      # Variable in function should be lowercase
    "N818",      # Exception name should end in Error
    "G004",      # Logging statement uses f-string
    "COM812",    # Trailing comma (conflicts with formatter)
    "ISC001",    # Implicit string concatenation (conflicts with formatter)
    "TRY003",    # Long messages outside the exception class
    "EM101",     # Raw string in exception
    "EM102",     # f-string in exception
    "PLR2004",   # Magic value in comparison
    "S101",      # Use of assert
    "BLE001",    # Blind except
    "PLC0415",   # Import not at top level (allow lazy imports)
    "UP007",     # Use X | Y for type annotations (3.10+ compatibility)
    "PERF203",   # Try-except in loop
    "PLR0913",   # Too many arguments in function definition
    "PLR0915",   # Too many statements in function
    "PLR0912",   # Too many branches
    "PLR0911",   # Too many return statements (consistent with the other relaxed PLR complexity rules)
    "PLR0911",   # Too many return statements (consistent with the other relaxed PLR complexity rules)
    "PLW2901",   # Loop variable overwritten
    "PLW1510",   # subprocess.run without check (errors handled manually)
    "SLF001",    # Private member access
    "ARG001",    # Unused function argument
    "EXE001",    # Shebang without executable bit
    "SIM108",    # Use ternary operator (readability preference)
    "PIE810",    # startswith with tuple (readability)
    "PERF401",   # Use list.extend (readability preference)
    "TC003",     # Move stdlib import into type-checking block
]

[tool.ruff.lint.per-file-ignores]
# CLI/gate scripts print human-readable results to stdout (T201) and are run as
# standalone files, not imported as packages (INP001).
"scripts/**" = ["T201", "INP001"]
# Tier 3 dataset CLIs are standalone command-line tools that print
# human-readable progress and results to stdout by design.
"src/skillevaluator/tier3/generate_dataset.py" = ["T201"]
# Reference skill scripts are standalone executables run by the agent under
# Harbor; printing to stdout is their output contract. Shared helpers beside
# them are imported from the script directory, not as a package (INP001).
"src/skillevaluator/tier3/reference_skills/**" = ["T201", "INP001"]
# pytest test directories are implicit namespace packages by design (no
# __init__.py anywhere under tests/).
"tests/**" = ["INP001"]
# Test helper / fixture scripts run standalone and print to stdout by design.
"tests/golden/generate_cli_surface.py" = ["T201"]
"tests/fixtures/skills/**" = ["T201"]

[tool.ruff.lint.isort]
known-first-party = ["skillevaluator"]

[tool.pytest.ini_options]
testpaths = ["tests"]
# Put the repo root on sys.path so the release tooling tests can
# ``import scripts.release.*`` (the tooling is not part of the installed wheel).
pythonpath = ["."]
# Union of the file-discovery patterns, including *_test.py.
python_files = ["test_*.py", "*_test.py"]
python_classes = ["Test*"]
python_functions = ["test_*"]
# addopts + markers carried over so marker-based selection works
# and live/integration suites are excluded from the default fast loop.
addopts = "--tb=short -m 'not live and not integration'"
markers = [
    "integration: end-to-end integration tests (may require fixtures or longer runtime)",
    "live: live tests against the in-process Harbor backend (requires the tier3 extra + NVIDIA_API_KEY)",
]