Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(validation): report non-string YAML keys (#178)

Rio Yub25e02e678e9
diff --git a/CHANGELOG.md b/CHANGELOG.md
index eea07f3c..33ecb538 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -30,6 +30,9 @@ All notable changes to SkillEvaluator are documented in this file.
### Fixed
+- Report non-string YAML keys as validation errors in skills, rules, workflows,
+ and plugin manifests instead of raising a `TypeError`, including frontmatter
+ `metadata` keys. Show boolean, null, and date keys in a readable YAML form.
- Harbor ``result.json`` case ids now prefer canonical ``task_id.path`` metadata
over repository-prefixed ``task_name`` values when resolving eval entries.
- Codex log synthesis maps ``web_search`` action payloads and ``collab_tool_call``
diff --git a/src/skillevaluator/models/field_validators.py b/src/skillevaluator/models/field_validators.py
index 60f88662..3f3f52af 100644
--- a/src/skillevaluator/models/field_validators.py
+++ b/src/skillevaluator/models/field_validators.py
@@ -52,5 +52,5 @@ def parse_nested_model(model_class: type, v: Any, error_message: str | None = No
raise ValueError(error_message)
return None
if isinstance(v, dict):
- return model_class(**v)
+ return model_class.model_validate(v)
return v
diff --git a/src/skillevaluator/models/skill.py b/src/skillevaluator/models/skill.py
index 8ac6a96c..9915317a 100644
--- a/src/skillevaluator/models/skill.py
+++ b/src/skillevaluator/models/skill.py
@@ -22,6 +22,7 @@ from skillevaluator.constants import (
NAME_MIN_LENGTH,
RESERVED_SKILL_NAMES,
)
+from skillevaluator.models.field_validators import parse_nested_model
XML_TAG_RE = re.compile(r"</?[A-Za-z][A-Za-z0-9_-]*(?:\s[^<>]*)?/?>|</?[A-Za-z][A-Za-z0-9_-]*\s[^<>]*$")
@@ -170,11 +171,7 @@ class SkillFrontmatter(BaseModel):
@classmethod
def parse_metadata(cls, v: Any) -> SkillMetadata | None:
"""Parse metadata dict into SkillMetadata model."""
- if v is None:
- return None
- if isinstance(v, dict):
- return SkillMetadata(**v)
- return v
+ return parse_nested_model(SkillMetadata, v)
@model_validator(mode="before")
@classmethod
diff --git a/src/skillevaluator/validators/frontmatter_parser.py b/src/skillevaluator/validators/frontmatter_parser.py
index a1aeac18..bb54aa9f 100644
--- a/src/skillevaluator/validators/frontmatter_parser.py
+++ b/src/skillevaluator/validators/frontmatter_parser.py
@@ -9,6 +9,7 @@ eliminating duplication across RulesSchemaValidator and WorkflowsSchemaValidator
import re
from dataclasses import dataclass
+from datetime import date
from pathlib import Path
from typing import Any
@@ -77,6 +78,20 @@ def parse_frontmatter(file_path: Path) -> tuple[ParsedFrontmatter | None, Valida
), result
+def format_validation_location(error: dict[str, Any]) -> str:
+ """Keep boolean, null, and date YAML keys recognizable in error locations."""
+ location = [str(loc) for loc in error["loc"]]
+ if error.get("type") == "invalid_key" and location:
+ key = error["input"]
+ if key is None:
+ location[-1] = "null"
+ elif isinstance(key, bool):
+ location[-1] = str(key).lower()
+ elif isinstance(key, date):
+ location[-1] = key.isoformat()
+ return ".".join(location)
+
+
def validate_pydantic_model(
model_class: type,
data: dict[str, Any],
@@ -95,9 +110,9 @@ def validate_pydantic_model(
from pydantic import ValidationError
try:
- return model_class(**data)
+ return model_class.model_validate(data)
except ValidationError as e:
for error in e.errors():
- field = ".".join(str(loc) for loc in error["loc"])
+ field = format_validation_location(error)
result.add_error(f"Field '{field}': {error['msg']}")
return None
diff --git a/src/skillevaluator/validators/plugin_schema.py b/src/skillevaluator/validators/plugin_schema.py
index 84d1aa7a..e57bcf2b 100644
--- a/src/skillevaluator/validators/plugin_schema.py
+++ b/src/skillevaluator/validators/plugin_schema.py
@@ -34,6 +34,7 @@ from skillevaluator.logging_config import get_logger
from skillevaluator.models.plugin import PluginManifest
from skillevaluator.models.result import Finding, Severity, ValidationResult
from skillevaluator.validators.base import ValidatorBase
+from skillevaluator.validators.frontmatter_parser import format_validation_location
logger = get_logger(__name__)
@@ -93,7 +94,7 @@ class PluginSchemaValidator(ValidatorBase):
data = self._load_yaml(manifest_path, result)
if data is not None:
try:
- manifest = PluginManifest(**data)
+ manifest = PluginManifest.model_validate(data)
except ValidationError as exc:
self._add_validation_findings(exc, manifest_path, result)
else:
@@ -187,7 +188,7 @@ class PluginSchemaValidator(ValidatorBase):
) -> None:
"""Translate a pydantic ``ValidationError`` into structured findings."""
for error in exc.errors():
- location = ".".join(str(loc) for loc in error["loc"]) or "<root>"
+ location = format_validation_location(error) or "<root>"
error_type = error.get("type", "value_error")
check_name = f"schema:{location}:{error_type}"
result.add_finding(
diff --git a/src/skillevaluator/validators/schema.py b/src/skillevaluator/validators/schema.py
index 26d2bbf2..291f6c1a 100644
--- a/src/skillevaluator/validators/schema.py
+++ b/src/skillevaluator/validators/schema.py
@@ -28,7 +28,7 @@ from skillevaluator.logging_config import get_logger
from skillevaluator.models.result import Finding, Severity
from skillevaluator.models.skill import SkillFrontmatter, SkillManifest
from skillevaluator.validators.base import ValidationResult, ValidatorBase
-from skillevaluator.validators.frontmatter_parser import FRONTMATTER_PATTERN
+from skillevaluator.validators.frontmatter_parser import FRONTMATTER_PATTERN, format_validation_location
from skillevaluator.validators.policy import ValidationPolicy, default_policy
logger = get_logger(__name__)
@@ -244,7 +244,7 @@ class SchemaValidator(ValidatorBase):
return result
try:
- frontmatter = SkillFrontmatter(**data)
+ frontmatter = SkillFrontmatter.model_validate(data)
result.add_success(
check_name="frontmatter_valid",
message=f"Valid frontmatter for skill '{frontmatter.name}'",
@@ -259,7 +259,7 @@ class SchemaValidator(ValidatorBase):
)
except ValidationError as e:
for error in e.errors():
- field = ".".join(str(loc) for loc in error["loc"])
+ field = format_validation_location(error)
result.add_finding(
Finding(
category="SCHEMA",
diff --git a/tests/validators/test_plugin_schema.py b/tests/validators/test_plugin_schema.py
index cc170698..eac0fba5 100644
--- a/tests/validators/test_plugin_schema.py
+++ b/tests/validators/test_plugin_schema.py
@@ -5,6 +5,8 @@
from pathlib import Path
+import pytest
+
from skillevaluator.constants import PLUGIN_MANIFEST_TYPE, PLUGIN_MODE
from skillevaluator.validators.plugin_schema import PluginSchemaValidator
@@ -30,6 +32,26 @@ def _write_manifest(dir_path: Path, body: str, name: str = "agent_plugin.yaml")
class TestPluginSchemaValidator:
+ @pytest.mark.parametrize("key", ["123", "true", "false", "null", "2026-01-01"])
+ def test_non_string_key_returns_validation_finding(self, tmp_path, key):
+ manifest = _write_manifest(tmp_path, f"{key}: extra value\n{_VALID_MANIFEST}")
+
+ result = PluginSchemaValidator().validate(tmp_path)
+
+ assert not result.passed
+ finding = next(finding for finding in result.findings if finding.check_name == f"schema:{key}:invalid_key")
+ assert finding.message == f"Field '{key}': Keys should be strings"
+ assert finding.file_path == str(manifest)
+
+ def test_quoted_unknown_key_remains_an_extra_field_error(self, tmp_path):
+ _write_manifest(tmp_path, f'"true": extra value\n{_VALID_MANIFEST}')
+
+ result = PluginSchemaValidator().validate(tmp_path)
+
+ assert not result.passed
+ assert any(finding.check_name == "schema:true:extra_forbidden" for finding in result.findings)
+ assert not any("invalid_key" in finding.check_name for finding in result.findings)
+
def test_valid_manifest_passes_with_metadata(self, tmp_path: Path):
_write_manifest(tmp_path, _VALID_MANIFEST)
result = PluginSchemaValidator().validate(tmp_path)
diff --git a/tests/validators/test_rules_schema.py b/tests/validators/test_rules_schema.py
index b758fb0d..1c77f866 100644
--- a/tests/validators/test_rules_schema.py
+++ b/tests/validators/test_rules_schema.py
@@ -97,6 +97,35 @@ description: unclosed quote
assert result.passed, f"Expected validation to pass. Errors: {result.errors}"
assert any("python-standards" in msg.lower() for msg in result.messages)
+ @pytest.mark.parametrize("key", ["123", "true", "false", "null", "2026-01-01"])
+ def test_non_string_frontmatter_key_returns_validation_error(self, validator, valid_rule_file, key):
+ content = valid_rule_file.read_text()
+ valid_rule_file.write_text(content.replace("---\n", f"---\n{key}: extra value\n", 1))
+
+ result = validator.validate(valid_rule_file)
+
+ assert not result.passed
+ assert any(f"Field '{key}': Keys should be strings" in error for error in result.errors)
+
+ def test_string_extra_frontmatter_key_is_preserved(self, validator, valid_rule_file):
+ content = valid_rule_file.read_text()
+ valid_rule_file.write_text(content.replace("---\n", '---\n"123": extra value\n', 1))
+
+ result = validator.validate(valid_rule_file)
+
+ assert result.passed, result.errors
+ assert result.metadata["frontmatter"].model_extra["123"] == "extra value"
+
+ @pytest.mark.parametrize("key", ["123", "true", "false", "null", "2026-01-01"])
+ def test_non_string_metadata_key_returns_validation_error(self, validator, valid_rule_file, key):
+ content = valid_rule_file.read_text()
+ valid_rule_file.write_text(content.replace("metadata:\n", f"metadata:\n {key}: extra value\n", 1))
+
+ result = validator.validate(valid_rule_file)
+
+ assert not result.passed
+ assert any(f"Field 'metadata.{key}': Keys should be strings" in error for error in result.errors)
+
def test_validate_missing_required_fields(
self, validator: RulesSchemaValidator, invalid_rule_missing_required: Path
):
diff --git a/tests/validators/test_schema.py b/tests/validators/test_schema.py
index 43474afe..0cf14883 100644
--- a/tests/validators/test_schema.py
+++ b/tests/validators/test_schema.py
@@ -8,12 +8,43 @@ Based on SkillEvaluator HOW_TO_CONTRIBUTE_SKILLS.md specification.
from pathlib import Path
+import pytest
+
from skillevaluator.validators.schema import SchemaValidator
class TestSchemaValidator:
"""Test cases for SchemaValidator based on SkillEvaluator spec."""
+ @pytest.mark.parametrize("key", ["123", "true", "false", "null", "2026-01-01"])
+ @pytest.mark.parametrize("nested", [False, True])
+ def test_non_string_key_returns_validation_error(self, sample_skill_dir, key, nested):
+ manifest = sample_skill_dir / "SKILL.md"
+ content = manifest.read_text()
+ marker = "metadata:\n" if nested else "---\n"
+ indentation = " " if nested else ""
+ manifest.write_text(content.replace(marker, f"{marker}{indentation}{key}: extra value\n", 1))
+
+ result = SchemaValidator().validate(sample_skill_dir)
+
+ assert not result.passed
+ field = f"metadata.{key}" if nested else key
+ findings = [finding for finding in result.findings if finding.check_name == "frontmatter_field"]
+ assert any(finding.message == f"Field '{field}': Keys should be strings" for finding in findings)
+ assert any(finding.metadata == {"field": field, "error_type": "invalid_key"} for finding in findings)
+
+ @pytest.mark.parametrize("key", ["123", "true", "false", "null", "2026-01-01"])
+ def test_quoted_extra_keys_remain_strings(self, sample_skill_dir, key):
+ manifest = sample_skill_dir / "SKILL.md"
+ content = manifest.read_text().replace("---\n", f'---\n"{key}": top-level\n', 1)
+ manifest.write_text(content.replace("metadata:\n", f'metadata:\n "{key}": nested\n', 1))
+
+ result = SchemaValidator().validate(sample_skill_dir)
+
+ assert result.passed, result.errors
+ assert result.metadata["frontmatter"].model_extra[key] == "top-level"
+ assert result.metadata["frontmatter"].metadata.model_extra[key] == "nested"
+
def test_valid_skill(self, sample_skill_dir: Path):
"""Test validation passes for valid skill with proper SkillEvaluator structure."""
validator = SchemaValidator()
diff --git a/tests/validators/test_workflows_schema.py b/tests/validators/test_workflows_schema.py
index f159c606..1e6533d8 100644
--- a/tests/validators/test_workflows_schema.py
+++ b/tests/validators/test_workflows_schema.py
@@ -169,6 +169,31 @@ metadata:
assert result.passed, f"Expected validation to pass. Errors: {result.errors}"
assert any("fastapi-setup" in msg.lower() for msg in result.messages)
+ @pytest.mark.parametrize("relative_path", ["workflow-rules.mdc", "references/api-design.mdc"])
+ @pytest.mark.parametrize("key", ["123", "true", "false", "null", "2026-01-01"])
+ def test_non_string_frontmatter_key_returns_validation_error(
+ self, validator, valid_workflow_dir, relative_path, key
+ ):
+ manifest = valid_workflow_dir / relative_path
+ content = manifest.read_text()
+ manifest.write_text(content.replace("---\n", f"---\n{key}: extra value\n", 1))
+
+ result = validator.validate(valid_workflow_dir)
+
+ assert not result.passed
+ assert any(f"Field '{key}': Keys should be strings" in error for error in result.errors)
+
+ @pytest.mark.parametrize("key", ["123", "true", "false", "null", "2026-01-01"])
+ def test_non_string_metadata_key_returns_validation_error(self, validator, valid_workflow_dir, key):
+ manifest = valid_workflow_dir / "workflow-rules.mdc"
+ content = manifest.read_text()
+ manifest.write_text(content.replace("metadata:\n", f"metadata:\n {key}: extra value\n", 1))
+
+ result = validator.validate(valid_workflow_dir)
+
+ assert not result.passed
+ assert any(f"Field 'metadata.{key}': Keys should be strings" in error for error in result.errors)
+
def test_validate_missing_readme(self, validator: WorkflowsSchemaValidator, incomplete_workflow_dir: Path):
"""Test validation fails when README.md is missing."""
result = validator.validate(incomplete_workflow_dir)