Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(validation): report non-string YAML keys (#178)
Rio Yub25e02e678e9
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex eea07f3c..33ecb538 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -30,6 +30,9 @@ All notable changes to SkillEvaluator are documented in this file.### Fixed+- Report non-string YAML keys as validation errors in skills, rules, workflows,+ and plugin manifests instead of raising a `TypeError`, including frontmatter+ `metadata` keys. Show boolean, null, and date keys in a readable YAML form.- Harbor ``result.json`` case ids now prefer canonical ``task_id.path`` metadataover repository-prefixed ``task_name`` values when resolving eval entries.- Codex log synthesis maps ``web_search`` action payloads and ``collab_tool_call``diff --git a/src/skillevaluator/models/field_validators.py b/src/skillevaluator/models/field_validators.pyindex 60f88662..3f3f52af 100644--- a/src/skillevaluator/models/field_validators.py+++ b/src/skillevaluator/models/field_validators.py@@ -52,5 +52,5 @@ def parse_nested_model(model_class: type, v: Any, error_message: str | None = Noraise ValueError(error_message)return Noneif isinstance(v, dict):- return model_class(**v)+ return model_class.model_validate(v)return vdiff --git a/src/skillevaluator/models/skill.py b/src/skillevaluator/models/skill.pyindex 8ac6a96c..9915317a 100644--- a/src/skillevaluator/models/skill.py+++ b/src/skillevaluator/models/skill.py@@ -22,6 +22,7 @@ from skillevaluator.constants import (NAME_MIN_LENGTH,RESERVED_SKILL_NAMES,)+from skillevaluator.models.field_validators import parse_nested_modelXML_TAG_RE = re.compile(r"</?[A-Za-z][A-Za-z0-9_-]*(?:\s[^<>]*)?/?>|</?[A-Za-z][A-Za-z0-9_-]*\s[^<>]*$")@@ -170,11 +171,7 @@ class SkillFrontmatter(BaseModel):@classmethoddef parse_metadata(cls, v: Any) -> SkillMetadata | None:"""Parse metadata dict into SkillMetadata model."""- if v is None:- return None- if isinstance(v, dict):- return SkillMetadata(**v)- return v+ return parse_nested_model(SkillMetadata, v)@model_validator(mode="before")@classmethoddiff --git a/src/skillevaluator/validators/frontmatter_parser.py b/src/skillevaluator/validators/frontmatter_parser.pyindex a1aeac18..bb54aa9f 100644--- a/src/skillevaluator/validators/frontmatter_parser.py+++ b/src/skillevaluator/validators/frontmatter_parser.py@@ -9,6 +9,7 @@ eliminating duplication across RulesSchemaValidator and WorkflowsSchemaValidatorimport refrom dataclasses import dataclass+from datetime import datefrom pathlib import Pathfrom typing import Any@@ -77,6 +78,20 @@ def parse_frontmatter(file_path: Path) -> tuple[ParsedFrontmatter | None, Valida), result+def format_validation_location(error: dict[str, Any]) -> str:+ """Keep boolean, null, and date YAML keys recognizable in error locations."""+ location = [str(loc) for loc in error["loc"]]+ if error.get("type") == "invalid_key" and location:+ key = error["input"]+ if key is None:+ location[-1] = "null"+ elif isinstance(key, bool):+ location[-1] = str(key).lower()+ elif isinstance(key, date):+ location[-1] = key.isoformat()+ return ".".join(location)++def validate_pydantic_model(model_class: type,data: dict[str, Any],@@ -95,9 +110,9 @@ def validate_pydantic_model(from pydantic import ValidationErrortry:- return model_class(**data)+ return model_class.model_validate(data)except ValidationError as e:for error in e.errors():- field = ".".join(str(loc) for loc in error["loc"])+ field = format_validation_location(error)result.add_error(f"Field '{field}': {error['msg']}")return Nonediff --git a/src/skillevaluator/validators/plugin_schema.py b/src/skillevaluator/validators/plugin_schema.pyindex 84d1aa7a..e57bcf2b 100644--- a/src/skillevaluator/validators/plugin_schema.py+++ b/src/skillevaluator/validators/plugin_schema.py@@ -34,6 +34,7 @@ from skillevaluator.logging_config import get_loggerfrom skillevaluator.models.plugin import PluginManifestfrom skillevaluator.models.result import Finding, Severity, ValidationResultfrom skillevaluator.validators.base import ValidatorBase+from skillevaluator.validators.frontmatter_parser import format_validation_locationlogger = get_logger(__name__)@@ -93,7 +94,7 @@ class PluginSchemaValidator(ValidatorBase):data = self._load_yaml(manifest_path, result)if data is not None:try:- manifest = PluginManifest(**data)+ manifest = PluginManifest.model_validate(data)except ValidationError as exc:self._add_validation_findings(exc, manifest_path, result)else:@@ -187,7 +188,7 @@ class PluginSchemaValidator(ValidatorBase):) -> None:"""Translate a pydantic ``ValidationError`` into structured findings."""for error in exc.errors():- location = ".".join(str(loc) for loc in error["loc"]) or "<root>"+ location = format_validation_location(error) or "<root>"error_type = error.get("type", "value_error")check_name = f"schema:{location}:{error_type}"result.add_finding(diff --git a/src/skillevaluator/validators/schema.py b/src/skillevaluator/validators/schema.pyindex 26d2bbf2..291f6c1a 100644--- a/src/skillevaluator/validators/schema.py+++ b/src/skillevaluator/validators/schema.py@@ -28,7 +28,7 @@ from skillevaluator.logging_config import get_loggerfrom skillevaluator.models.result import Finding, Severityfrom skillevaluator.models.skill import SkillFrontmatter, SkillManifestfrom skillevaluator.validators.base import ValidationResult, ValidatorBase-from skillevaluator.validators.frontmatter_parser import FRONTMATTER_PATTERN+from skillevaluator.validators.frontmatter_parser import FRONTMATTER_PATTERN, format_validation_locationfrom skillevaluator.validators.policy import ValidationPolicy, default_policylogger = get_logger(__name__)@@ -244,7 +244,7 @@ class SchemaValidator(ValidatorBase):return resulttry:- frontmatter = SkillFrontmatter(**data)+ frontmatter = SkillFrontmatter.model_validate(data)result.add_success(check_name="frontmatter_valid",message=f"Valid frontmatter for skill '{frontmatter.name}'",@@ -259,7 +259,7 @@ class SchemaValidator(ValidatorBase):)except ValidationError as e:for error in e.errors():- field = ".".join(str(loc) for loc in error["loc"])+ field = format_validation_location(error)result.add_finding(Finding(category="SCHEMA",diff --git a/tests/validators/test_plugin_schema.py b/tests/validators/test_plugin_schema.pyindex cc170698..eac0fba5 100644--- a/tests/validators/test_plugin_schema.py+++ b/tests/validators/test_plugin_schema.py@@ -5,6 +5,8 @@from pathlib import Path+import pytest+from skillevaluator.constants import PLUGIN_MANIFEST_TYPE, PLUGIN_MODEfrom skillevaluator.validators.plugin_schema import PluginSchemaValidator@@ -30,6 +32,26 @@ def _write_manifest(dir_path: Path, body: str, name: str = "agent_plugin.yaml")class TestPluginSchemaValidator:+ @pytest.mark.parametrize("key", ["123", "true", "false", "null", "2026-01-01"])+ def test_non_string_key_returns_validation_finding(self, tmp_path, key):+ manifest = _write_manifest(tmp_path, f"{key}: extra value\n{_VALID_MANIFEST}")++ result = PluginSchemaValidator().validate(tmp_path)++ assert not result.passed+ finding = next(finding for finding in result.findings if finding.check_name == f"schema:{key}:invalid_key")+ assert finding.message == f"Field '{key}': Keys should be strings"+ assert finding.file_path == str(manifest)++ def test_quoted_unknown_key_remains_an_extra_field_error(self, tmp_path):+ _write_manifest(tmp_path, f'"true": extra value\n{_VALID_MANIFEST}')++ result = PluginSchemaValidator().validate(tmp_path)++ assert not result.passed+ assert any(finding.check_name == "schema:true:extra_forbidden" for finding in result.findings)+ assert not any("invalid_key" in finding.check_name for finding in result.findings)+def test_valid_manifest_passes_with_metadata(self, tmp_path: Path):_write_manifest(tmp_path, _VALID_MANIFEST)result = PluginSchemaValidator().validate(tmp_path)diff --git a/tests/validators/test_rules_schema.py b/tests/validators/test_rules_schema.pyindex b758fb0d..1c77f866 100644--- a/tests/validators/test_rules_schema.py+++ b/tests/validators/test_rules_schema.py@@ -97,6 +97,35 @@ description: unclosed quoteassert result.passed, f"Expected validation to pass. Errors: {result.errors}"assert any("python-standards" in msg.lower() for msg in result.messages)+ @pytest.mark.parametrize("key", ["123", "true", "false", "null", "2026-01-01"])+ def test_non_string_frontmatter_key_returns_validation_error(self, validator, valid_rule_file, key):+ content = valid_rule_file.read_text()+ valid_rule_file.write_text(content.replace("---\n", f"---\n{key}: extra value\n", 1))++ result = validator.validate(valid_rule_file)++ assert not result.passed+ assert any(f"Field '{key}': Keys should be strings" in error for error in result.errors)++ def test_string_extra_frontmatter_key_is_preserved(self, validator, valid_rule_file):+ content = valid_rule_file.read_text()+ valid_rule_file.write_text(content.replace("---\n", '---\n"123": extra value\n', 1))++ result = validator.validate(valid_rule_file)++ assert result.passed, result.errors+ assert result.metadata["frontmatter"].model_extra["123"] == "extra value"++ @pytest.mark.parametrize("key", ["123", "true", "false", "null", "2026-01-01"])+ def test_non_string_metadata_key_returns_validation_error(self, validator, valid_rule_file, key):+ content = valid_rule_file.read_text()+ valid_rule_file.write_text(content.replace("metadata:\n", f"metadata:\n {key}: extra value\n", 1))++ result = validator.validate(valid_rule_file)++ assert not result.passed+ assert any(f"Field 'metadata.{key}': Keys should be strings" in error for error in result.errors)+def test_validate_missing_required_fields(self, validator: RulesSchemaValidator, invalid_rule_missing_required: Path):diff --git a/tests/validators/test_schema.py b/tests/validators/test_schema.pyindex 43474afe..0cf14883 100644--- a/tests/validators/test_schema.py+++ b/tests/validators/test_schema.py@@ -8,12 +8,43 @@ Based on SkillEvaluator HOW_TO_CONTRIBUTE_SKILLS.md specification.from pathlib import Path+import pytest+from skillevaluator.validators.schema import SchemaValidatorclass TestSchemaValidator:"""Test cases for SchemaValidator based on SkillEvaluator spec."""+ @pytest.mark.parametrize("key", ["123", "true", "false", "null", "2026-01-01"])+ @pytest.mark.parametrize("nested", [False, True])+ def test_non_string_key_returns_validation_error(self, sample_skill_dir, key, nested):+ manifest = sample_skill_dir / "SKILL.md"+ content = manifest.read_text()+ marker = "metadata:\n" if nested else "---\n"+ indentation = " " if nested else ""+ manifest.write_text(content.replace(marker, f"{marker}{indentation}{key}: extra value\n", 1))++ result = SchemaValidator().validate(sample_skill_dir)++ assert not result.passed+ field = f"metadata.{key}" if nested else key+ findings = [finding for finding in result.findings if finding.check_name == "frontmatter_field"]+ assert any(finding.message == f"Field '{field}': Keys should be strings" for finding in findings)+ assert any(finding.metadata == {"field": field, "error_type": "invalid_key"} for finding in findings)++ @pytest.mark.parametrize("key", ["123", "true", "false", "null", "2026-01-01"])+ def test_quoted_extra_keys_remain_strings(self, sample_skill_dir, key):+ manifest = sample_skill_dir / "SKILL.md"+ content = manifest.read_text().replace("---\n", f'---\n"{key}": top-level\n', 1)+ manifest.write_text(content.replace("metadata:\n", f'metadata:\n "{key}": nested\n', 1))++ result = SchemaValidator().validate(sample_skill_dir)++ assert result.passed, result.errors+ assert result.metadata["frontmatter"].model_extra[key] == "top-level"+ assert result.metadata["frontmatter"].metadata.model_extra[key] == "nested"+def test_valid_skill(self, sample_skill_dir: Path):"""Test validation passes for valid skill with proper SkillEvaluator structure."""validator = SchemaValidator()diff --git a/tests/validators/test_workflows_schema.py b/tests/validators/test_workflows_schema.pyindex f159c606..1e6533d8 100644--- a/tests/validators/test_workflows_schema.py+++ b/tests/validators/test_workflows_schema.py@@ -169,6 +169,31 @@ metadata:assert result.passed, f"Expected validation to pass. Errors: {result.errors}"assert any("fastapi-setup" in msg.lower() for msg in result.messages)+ @pytest.mark.parametrize("relative_path", ["workflow-rules.mdc", "references/api-design.mdc"])+ @pytest.mark.parametrize("key", ["123", "true", "false", "null", "2026-01-01"])+ def test_non_string_frontmatter_key_returns_validation_error(+ self, validator, valid_workflow_dir, relative_path, key+ ):+ manifest = valid_workflow_dir / relative_path+ content = manifest.read_text()+ manifest.write_text(content.replace("---\n", f"---\n{key}: extra value\n", 1))++ result = validator.validate(valid_workflow_dir)++ assert not result.passed+ assert any(f"Field '{key}': Keys should be strings" in error for error in result.errors)++ @pytest.mark.parametrize("key", ["123", "true", "false", "null", "2026-01-01"])+ def test_non_string_metadata_key_returns_validation_error(self, validator, valid_workflow_dir, key):+ manifest = valid_workflow_dir / "workflow-rules.mdc"+ content = manifest.read_text()+ manifest.write_text(content.replace("metadata:\n", f"metadata:\n {key}: extra value\n", 1))++ result = validator.validate(valid_workflow_dir)++ assert not result.passed+ assert any(f"Field 'metadata.{key}': Keys should be strings" in error for error in result.errors)+def test_validate_missing_readme(self, validator: WorkflowsSchemaValidator, incomplete_workflow_dir: Path):"""Test validation fails when README.md is missing."""result = validator.validate(incomplete_workflow_dir)