Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(tier1): report a nested SKILL.md once on case-insensitive filesystems (#599)
Christopher Kevina8a67e4c69d8
diff --git a/src/skillevaluator/validators/schema.py b/src/skillevaluator/validators/schema.pyindex bc78ef8b..be954346 100644--- a/src/skillevaluator/validators/schema.py+++ b/src/skillevaluator/validators/schema.py@@ -772,11 +772,13 @@ class SchemaValidator(ValidatorBase):break# Same folders the harness check skips: evals/ and versions/ are scanned like any other folder.+ # Match listed names: on a case-insensitive filesystem a lookup by name answers for both+ # spellings, which reported one nested manifest twice.nested = sorted(path- for name in SKILL_MANIFEST_VARIANTS- for path in skill_path.rglob(name)- if path.parent != skill_path+ for path in skill_path.rglob("*")+ if path.name in SKILL_MANIFEST_VARIANTS+ and path.parent != skill_pathand not any(part in SKILL_NOT_SHIPPED_DIRS for part in path.relative_to(skill_path).parts))for path in nested:diff --git a/tests/validators/test_schema.py b/tests/validators/test_schema.pyindex dd041fd9..a4e62279 100644--- a/tests/validators/test_schema.py+++ b/tests/validators/test_schema.py@@ -1657,6 +1657,16 @@ class TestSchemaPortabilityChecks:assert len(findings) == 1assert "references/inner/SKILL.md" in findings[0].message+ def test_nested_lowercase_manifest_is_reported_once_with_its_own_name(self, tmp_path: Path):+ """On macOS and Windows one nested file answers to both SKILL.md and skill.md."""+ skill_dir = _write_skill(tmp_path, _BASE_FM)+ (skill_dir / "references" / "inner").mkdir(parents=True)+ (skill_dir / "references" / "inner" / "skill.md").write_text("---\nname: inner\n---\n")++ findings = _checks(SchemaValidator().validate(skill_dir), "nested_manifest")++ assert [f.file_path for f in findings] == [str(skill_dir / "references" / "inner" / "skill.md")]+def test_nested_manifest_under_evals_is_reported(self, tmp_path: Path):skill_dir = _write_skill(tmp_path, _BASE_FM)(skill_dir / "evals" / "snapshot").mkdir(parents=True)