Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Tests for skillevaluator.validators.frontmatter_parser module."""
from pathlib import Path
import pytest
from pydantic import BaseModel
from skillevaluator.validators.base import ValidationResult
from skillevaluator.validators.frontmatter_parser import parse_frontmatter, validate_pydantic_model
class TestParseFrontmatter:
"""Tests for parse_frontmatter utility."""
def test_valid_frontmatter(self, tmp_path: Path):
"""Test parsing valid frontmatter."""
test_file = tmp_path / "test.mdc"
test_file.write_text("""---
title: Test
description: A test file
---
# Content here
""")
parsed, result = parse_frontmatter(test_file)
assert parsed is not None
assert result.passed
assert parsed.yaml_data["title"] == "Test"
assert parsed.content.strip() == "# Content here"
@pytest.mark.parametrize("newline", ["\n", "\r\n"], ids=["lf", "crlf"])
def test_first_body_line_indentation_is_preserved(self, tmp_path: Path, newline: str):
test_file = tmp_path / "indented-body.mdc"
content = newline.join(["---", "title: Test", "---", " [code](false.md)", ""])
test_file.write_bytes(content.encode())
parsed, result = parse_frontmatter(test_file)
assert parsed is not None
assert result.passed
assert parsed.content == " [code](false.md)\n"
def test_utf8_bom_is_accepted(self, tmp_path: Path):
"""A valid frontmatter file that starts with a UTF-8 BOM still parses (#91)."""
test_file = tmp_path / "bom.md"
body = """---
title: Bom
description: A file whose only extra is a leading UTF-8 BOM
---
# Content
"""
test_file.write_bytes(b"\xef\xbb\xbf" + body.encode("utf-8"))
parsed, result = parse_frontmatter(test_file)
assert parsed is not None
assert result.passed
assert parsed.yaml_data["title"] == "Bom"
assert parsed.content.strip() == "# Content"
def test_missing_frontmatter(self, tmp_path: Path):
"""Test file without frontmatter markers."""
test_file = tmp_path / "test.mdc"
test_file.write_text("# Just content, no frontmatter")
parsed, result = parse_frontmatter(test_file)
assert parsed is None
assert not result.passed
assert any("frontmatter" in err.lower() for err in result.errors)
def test_invalid_yaml(self, tmp_path: Path):
"""Test file with invalid YAML in frontmatter."""
test_file = tmp_path / "test.mdc"
test_file.write_text("""---
title: [invalid yaml
missing bracket
---
# Content
""")
parsed, result = parse_frontmatter(test_file)
assert parsed is None
assert not result.passed
assert any("yaml" in err.lower() for err in result.errors)
def test_empty_frontmatter(self, tmp_path: Path):
"""Test file with empty frontmatter."""
test_file = tmp_path / "test.mdc"
test_file.write_text("""---
---
# Content
""")
parsed, result = parse_frontmatter(test_file)
assert parsed is None
assert not result.passed
def test_nonexistent_file(self, tmp_path: Path):
"""Test parsing nonexistent file."""
test_file = tmp_path / "nonexistent.mdc"
parsed, result = parse_frontmatter(test_file)
assert parsed is None
assert not result.passed
class TestValidatePydanticModel:
"""Tests for validate_pydantic_model utility."""
def test_valid_data(self):
"""Test validation with valid data."""
class TestModel(BaseModel):
name: str
count: int
result = ValidationResult()
model = validate_pydantic_model(TestModel, {"name": "test", "count": 5}, result)
assert model is not None
assert result.passed
assert model.name == "test"
def test_invalid_data(self):
"""Test validation with invalid data."""
class TestModel(BaseModel):
name: str
count: int
result = ValidationResult()
model = validate_pydantic_model(TestModel, {"name": "test"}, result) # Missing count
assert model is None
assert not result.passed
assert any("count" in err.lower() for err in result.errors)