Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""The staged Harbor verifier must run on the task image's Python 3.9."""
from __future__ import annotations
import ast
from pathlib import Path
import pytest
from skillevaluator.tier3.harbor.adapter import _copy_verifier
def _staged_modules(tmp_path: Path) -> list[Path]:
_copy_verifier(tmp_path)
modules = sorted((tmp_path / "tests/skill_evaluator").rglob("*.py"))
assert modules, "the verifier staged no Python modules"
return modules
def test_staged_verifier_parses_as_python39(tmp_path):
for module in _staged_modules(tmp_path):
source = module.read_text(encoding="utf-8")
try:
ast.parse(source, filename=str(module), feature_version=(3, 9))
except SyntaxError as error: # pragma: no cover - the failure message is the point
pytest.fail(f"{module.name} does not parse as Python 3.9: {error}")
def test_staged_verifier_calls_no_zip_with_strict(tmp_path):
# zip(..., strict=True) is new in Python 3.10; on 3.9 it raises
# "TypeError: zip() takes no keyword arguments" when the call runs.
offenders = []
for module in _staged_modules(tmp_path):
tree = ast.parse(module.read_text(encoding="utf-8"), filename=str(module))
for node in ast.walk(tree):
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id == "zip"
and any(keyword.arg == "strict" for keyword in node.keywords)
):
offenders.append(f"{module.name}:{node.lineno}")
assert offenders == []