Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""A historical baseline must use its own shared parser without import leaks."""
from __future__ import annotations
import subprocess
import sys
import pytest
from scripts import script_invocation_differential as differential
def test_current_git_baseline_loads_its_own_shared_modules(monkeypatch):
if subprocess.run(
["git", "rev-parse", "--verify", "HEAD"], cwd=differential.REPO_ROOT, capture_output=True
).returncode:
pytest.skip("source archive has no git baseline")
previous_path = sys.path.copy()
previous_modules = sys.modules.copy()
monkeypatch.setattr(
differential.host_checks._script_invocation, "_cmd_executes_script", lambda *_args, **_kwargs: False
)
monkeypatch.setattr(differential.host_checks._shell_lexing, "_shell_tokens", lambda _command: [])
host, template = differential._load_baseline("HEAD")
other_host, other_template = differential._load_baseline("HEAD")
assert sys.path == previous_path
for name in (
"checks",
"baseline_template",
"script_invocation",
"shell_lexing",
"optional_reference",
"secret_redaction",
"skillevaluator.tier3.eval_core.script_invocation",
"skillevaluator.tier3.eval_core.shell_lexing",
"skillevaluator.tier3.eval_core.optional_reference",
):
assert sys.modules.get(name) is previous_modules.get(name)
assert host._cmd_executes_script is not differential.host_checks._cmd_executes_script
assert template._cmd_executes_script is host._cmd_executes_script
assert host._cmd_executes_script is not other_host._cmd_executes_script
assert other_template._cmd_executes_script is other_host._cmd_executes_script
for command, expected in [("python calc.py", True), ("cat calc.py", False), ("sh -c 'python calc.py'", True)]:
assert host._cmd_executes_script(command, "calc.py") is expected
assert template._cmd_executes_script(command, "calc.py") is expected
assert other_host._cmd_executes_script(command, "calc.py") is expected
def test_invalid_git_baseline_does_not_change_imports():
previous_path = sys.path.copy()
previous_checker = sys.modules.get("checks")
with pytest.raises(subprocess.CalledProcessError):
differential._load_baseline("refs/invalid/missing-baseline")
assert sys.path == previous_path
assert sys.modules.get("checks") is previous_checker
def test_package_and_flat_import_eras_use_their_own_helpers(tmp_path, monkeypatch):
"""Two real refs cover both intermediate package imports and flat staging."""
def git(*args):
return subprocess.check_output(
["git", "-c", "user.name=Baseline test", "-c", "[email protected]", *args],
cwd=tmp_path,
text=True,
).strip()
git("init", "--quiet")
core = tmp_path / "src/skillevaluator/tier3/eval_core"
templates = tmp_path / "src/skillevaluator/tier3/harbor/templates"
core.mkdir(parents=True)
templates.mkdir(parents=True)
(templates / "eval.py").write_text("from checks import _cmd_executes_script\n")
(core / "checks.py").write_text(
"from skillevaluator.tier3.eval_core import script_invocation\n"
"_cmd_executes_script = script_invocation._cmd_executes_script\n"
)
parser = core / "script_invocation.py"
parser.write_text("def _cmd_executes_script(command, script): return True\n")
git("add", "src")
git("commit", "--quiet", "-m", "package imports")
package_ref = git("rev-parse", "HEAD")
(core / "checks.py").write_text(
"import script_invocation\n_cmd_executes_script = script_invocation._cmd_executes_script\n"
)
parser.write_text("def _cmd_executes_script(command, script): return False\n")
git("add", "src")
git("commit", "--quiet", "-m", "flat imports")
flat_ref = git("rev-parse", "HEAD")
monkeypatch.setattr(differential, "REPO_ROOT", tmp_path)
candidate = differential.host_checks._script_invocation
monkeypatch.setattr(candidate, "_cmd_executes_script", lambda *_args: None)
from skillevaluator.tier3 import eval_core
for ref, expected in [(package_ref, True), (flat_ref, False), (package_ref, True)]:
host, template = differential._load_baseline(ref)
assert host._cmd_executes_script("python calc.py", "calc.py") is expected
assert template._cmd_executes_script("python calc.py", "calc.py") is expected
assert eval_core.script_invocation is candidate
assert candidate._cmd_executes_script("python calc.py", "calc.py") is None