Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Tests for PluginSchemaValidator (bundle-reference plugin manifest validation)."""
from pathlib import Path
import pytest
from skillevaluator.constants import PLUGIN_MANIFEST_TYPE, PLUGIN_MODE
from skillevaluator.validators.plugin_schema import PluginSchemaValidator
_VALID_MANIFEST = """
name: my-bundle
description: A test bundle
version: 1.0
author:
email: [email protected]
skills:
refs:
- "github::example-org/example-repo::skills::build-infra"
mcp:
- name: filesystem
provider: stdio
"""
def _write_manifest(dir_path: Path, body: str, name: str = "agent_plugin.yaml") -> Path:
manifest = dir_path / name
manifest.write_text(body)
return manifest
class TestPluginSchemaValidator:
@pytest.mark.parametrize("key", ["123", "true", "false", "null", "2026-01-01"])
def test_non_string_key_returns_validation_finding(self, tmp_path, key):
manifest = _write_manifest(tmp_path, f"{key}: extra value\n{_VALID_MANIFEST}")
result = PluginSchemaValidator().validate(tmp_path)
assert not result.passed
finding = next(finding for finding in result.findings if finding.check_name == f"schema:{key}:invalid_key")
assert finding.message == f"Field '{key}': Keys should be strings"
assert finding.file_path == str(manifest)
def test_quoted_unknown_key_remains_an_extra_field_error(self, tmp_path):
_write_manifest(tmp_path, f'"true": extra value\n{_VALID_MANIFEST}')
result = PluginSchemaValidator().validate(tmp_path)
assert not result.passed
assert any(finding.check_name == "schema:true:extra_forbidden" for finding in result.findings)
assert not any("invalid_key" in finding.check_name for finding in result.findings)
def test_valid_manifest_passes_with_metadata(self, tmp_path: Path):
_write_manifest(tmp_path, _VALID_MANIFEST)
result = PluginSchemaValidator().validate(tmp_path)
assert result.passed
assert not result.findings
assert result.metadata["manifest_type"] == PLUGIN_MANIFEST_TYPE
assert result.metadata["plugin_mode"] == PLUGIN_MODE
assert result.metadata["plugin"]["name"] == "my-bundle"
def test_valid_manifest_file_path(self, tmp_path: Path):
manifest = _write_manifest(tmp_path, _VALID_MANIFEST)
result = PluginSchemaValidator().validate(manifest)
assert result.passed
def test_yml_extension_accepted(self, tmp_path: Path):
_write_manifest(tmp_path, _VALID_MANIFEST, name="agent_plugin.yml")
result = PluginSchemaValidator().validate(tmp_path)
assert result.passed
def test_missing_manifest_produces_finding(self, tmp_path: Path):
result = PluginSchemaValidator().validate(tmp_path)
assert not result.passed
assert any(f.check_name == "manifest_missing" for f in result.findings)
def test_invalid_yaml_produces_finding(self, tmp_path: Path):
_write_manifest(tmp_path, "name: [unclosed\n")
result = PluginSchemaValidator().validate(tmp_path)
assert not result.passed
assert any(f.check_name == "manifest_invalid_yaml" for f in result.findings)
def test_non_mapping_manifest_produces_finding(self, tmp_path: Path):
_write_manifest(tmp_path, "- just\n- a\n- list\n")
result = PluginSchemaValidator().validate(tmp_path)
assert not result.passed
assert any(f.check_name == "manifest_not_mapping" for f in result.findings)
def test_contract_violations_produce_schema_findings(self, tmp_path: Path):
_write_manifest(
tmp_path,
"""
name: bad-bundle
author:
email: not-an-email
skills:
refs:
- "badsource::noslash::x"
""",
)
result = PluginSchemaValidator().validate(tmp_path)
assert not result.passed
assert all(f.category == "PLUGIN_SCHEMA" for f in result.findings)
assert any(f.check_name.startswith("schema:") for f in result.findings)
def test_contained_plugin_with_name_passes(self, tmp_path: Path):
manifest = tmp_path / ".claude-plugin" / "plugin.json"
manifest.parent.mkdir()
manifest.write_text('{"name": "contained-plugin", "skills": ["demo"]}', encoding="utf-8")
result = PluginSchemaValidator().validate(tmp_path)
assert result.passed
assert result.metadata["manifest_type"] == "claude_plugin_json"
assert result.metadata["plugin_mode"] == "contained"
assert result.metadata["plugin"]["name"] == "contained-plugin"
def test_contained_plugin_requires_non_empty_name(self, tmp_path: Path):
manifest = tmp_path / ".claude-plugin" / "plugin.json"
manifest.parent.mkdir()
manifest.write_text('{"name": ""}', encoding="utf-8")
result = PluginSchemaValidator().validate(tmp_path)
assert not result.passed
assert any(f.check_name == "schema:name:missing" for f in result.findings)
def test_bundle_manifest_wins_over_contained_manifest(self, tmp_path: Path):
_write_manifest(tmp_path, _VALID_MANIFEST)
manifest = tmp_path / ".claude-plugin" / "plugin.json"
manifest.parent.mkdir()
manifest.write_text("not valid json", encoding="utf-8")
result = PluginSchemaValidator().validate(tmp_path)
assert result.passed
assert result.metadata["manifest_type"] == PLUGIN_MANIFEST_TYPE
def test_contained_plugin_merges_bundled_skill_schema_findings(self, tmp_path: Path):
manifest = tmp_path / ".claude-plugin" / "plugin.json"
manifest.parent.mkdir()
manifest.write_text('{"name": "contained-plugin"}', encoding="utf-8")
skill = tmp_path / "skills" / "broken-skill" / "SKILL.md"
skill.parent.mkdir(parents=True)
skill.write_text("# Missing frontmatter", encoding="utf-8")
result = PluginSchemaValidator().validate(tmp_path)
assert not result.passed
assert result.metadata["plugin"]["bundled_skills"] == ["broken-skill"]
assert any("[broken-skill]" in finding.file_path for finding in result.findings)
def test_contained_plugin_with_valid_bundled_skill_passes(self, tmp_path: Path):
manifest = tmp_path / ".claude-plugin" / "plugin.json"
manifest.parent.mkdir()
manifest.write_text('{"name": "contained-plugin"}', encoding="utf-8")
skill = tmp_path / "skills" / "demo" / "SKILL.md"
skill.parent.mkdir(parents=True)
skill.write_text(
"---\n"
"name: demo\n"
"description: Demo bundled skill\n"
"metadata:\n"
" author: Demo Author <[email protected]>\n"
"---\n"
"# Demo\n\n"
"## Instructions\nUse this demo skill.\n\n"
"## Examples\nRun the demo.\n",
encoding="utf-8",
)
result = PluginSchemaValidator().validate(tmp_path)
assert result.passed
assert any(detail.check_name == "demo" for detail in result.success_details)
def test_linked_skills_directory_produces_finding(self, tmp_path: Path):
_write_manifest(tmp_path, _VALID_MANIFEST)
target = tmp_path / "linked-skills"
target.mkdir()
skills_root = tmp_path / "skills"
skills_root.symlink_to(target, target_is_directory=True)
result = PluginSchemaValidator().validate(tmp_path)
assert not result.passed
finding = next(f for f in result.findings if f.check_name == "in_plugin_skill_discovery_error")
assert finding.category == "PLUGIN_SCHEMA"
assert finding.file_path == str(skills_root)
assert "unlinked directory" in finding.message
def test_skill_discovery_failure_produces_finding(self, tmp_path: Path, monkeypatch):
_write_manifest(tmp_path, _VALID_MANIFEST)
def fail_discovery(root):
raise ValueError("Cannot inspect skill manifest")
monkeypatch.setattr("skillevaluator.utils.helpers.find_bundled_plugin_skills", fail_discovery)
result = PluginSchemaValidator().validate(tmp_path)
assert not result.passed
finding = next(f for f in result.findings if f.check_name == "in_plugin_skill_discovery_error")
assert finding.category == "PLUGIN_SCHEMA"
assert finding.file_path == str(tmp_path / "skills")
assert "Cannot inspect skill manifest" in finding.message