Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(quality): score non-mapping metadata instead of crashing (#226)
Max Freedom Pollard1c435f74f0bf
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 9530474a..29b56661 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -68,6 +68,12 @@ All notable changes to SkillEvaluator are documented in this file.Harbor, so Docker Desktop finds its `compose` plugin, and passes`PYTHONUTF8` and `PYTHONIOENCODING` so Python UTF-8 mode reaches Harbor onnon-UTF-8 locales.+- A `SKILL.md` `metadata` value that is not a mapping, such as `true`, `1.5`, a+ date, or a list or string containing `author` or `tags`, no longer crashes+ the quality check with a `TypeError`, which aborted `validate`, `tier1`, and+ `quality-check` before any report was written. The schema check reports the+ malformed `metadata`, and the quality score counts `metadata.author` and+ `metadata.tags` as missing, as it already did for other non-mapping values.## 0.6.0 - 2026-10-20diff --git a/src/skillevaluator/validators/quality_score.py b/src/skillevaluator/validators/quality_score.pyindex ed27ce81..09061c2e 100644--- a/src/skillevaluator/validators/quality_score.py+++ b/src/skillevaluator/validators/quality_score.py@@ -867,7 +867,11 @@ class QualityScoreValidator(ValidatorBase):def _check_spec_fields(self, qs: QualityScoreResult, frontmatter: dict | None) -> None:if not frontmatter:return- metadata = frontmatter.get("metadata") or {}+ metadata = frontmatter.get("metadata")+ if not isinstance(metadata, dict):+ # A list, string, or scalar is invalid metadata per the schema; score it+ # as absent metadata rather than indexing into it.+ metadata = {}top_level_fields = {"version": 'Semantic version (e.g., "1.0.0")',diff --git a/tests/validators/test_quality_score.py b/tests/validators/test_quality_score.pyindex 127e3608..780e6694 100644--- a/tests/validators/test_quality_score.py+++ b/tests/validators/test_quality_score.py@@ -9,6 +9,7 @@ import pytestfrom skillevaluator.models.quality import QualityScoreResult, score_to_gradefrom skillevaluator.quality_config import load_quality_config, quality_config_digest+from skillevaluator.tier1.commands import run_validationfrom skillevaluator.validators.quality_score import QualityScoreValidator@@ -132,6 +133,23 @@ def _finding_messages(skill_dir: Path) -> list[str]:return [finding.message for finding in result.findings]+def _write_metadata_skill(tmp_path: Path, metadata: str) -> Path:+ """Write a skill whose frontmatter ``metadata`` is the given raw YAML value."""+ skill_dir = tmp_path / "odd-metadata"+ skill_dir.mkdir()+ (skill_dir / "SKILL.md").write_text(+ "---\n"+ "name: odd-metadata\n"+ "description: A focused skill. Use when testing malformed contributor metadata.\n"+ f"metadata: {metadata}\n"+ "---\n\n"+ "# Odd Metadata\n\n"+ "## Purpose\n\nThis skill verifies malformed metadata handling.\n\n"+ "## Instructions\n\n1. Check quality findings.\n\n"+ )+ return skill_dir++class TestScoreToGrade:def test_grade_a(self):assert score_to_grade(95.0) == "A"@@ -392,6 +410,28 @@ class TestQualityScoreValidator:assert any("metadata.author" in finding.message for finding in result.findings)+ @pytest.mark.parametrize("metadata", ["[author, tags]", '"author: Test User"', "true", "1.5", "2026-01-01"])+ def test_non_mapping_metadata_counts_as_missing_fields(self, tmp_path, metadata):+ """A list, string, or scalar ``metadata`` scores as absent metadata instead of being indexed."""+ skill_dir = _write_metadata_skill(tmp_path, metadata)++ result = QualityScoreValidator(min_score=0).validate(skill_dir)++ messages = [finding.message for finding in result.findings]+ assert "SKILL_SPEC recommended field missing: 'metadata.author'" in messages+ assert "SKILL_SPEC recommended field missing: 'metadata.tags'" in messages++ def test_non_mapping_metadata_is_reported_by_schema_not_a_crash(self, tmp_path):+ """The quality check must not abort Tier 1 before the schema finding for ``metadata`` is reported."""+ skill_dir = _write_metadata_skill(tmp_path, "[author, tags]")++ results = run_validation(skill_dir, checks="schema,quality", on_check=lambda _check: None)++ schema, quality = results+ assert not schema.passed+ assert any(finding.message.startswith("Field 'metadata':") for finding in schema.findings)+ assert quality.metadata["quality_scores"]["skill_name"] == "odd-metadata"+def test_xml_tags_in_description_remain_quality_error(self, tmp_path):"""Real XML/HTML-like tags in descriptions are still flagged."""skill_dir = tmp_path / "xml-desc"