Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(quality): address review parsing gaps

Christopher Kevin93f089fe3160
diff --git a/src/skillevaluator/models/skill.py b/src/skillevaluator/models/skill.py
index f780a1ad..8ac6a96c 100644
--- a/src/skillevaluator/models/skill.py
+++ b/src/skillevaluator/models/skill.py
@@ -23,7 +23,7 @@ from skillevaluator.constants import (
RESERVED_SKILL_NAMES,
)
-XML_TAG_RE = re.compile(r"</?[A-Za-z][A-Za-z0-9_-]*(?:\s[^<>]*)?>|</?[A-Za-z][A-Za-z0-9_-]*\s[^<>]*$")
+XML_TAG_RE = re.compile(r"</?[A-Za-z][A-Za-z0-9_-]*(?:\s[^<>]*)?/?>|</?[A-Za-z][A-Za-z0-9_-]*\s[^<>]*$")
#: Strict, bounded ASCII semantic-version pattern for optional metadata.version labels.
_SEMVER_COMPONENT = r"(?:0|[1-9][0-9]{0,8})"
diff --git a/src/skillevaluator/validators/quality_score.py b/src/skillevaluator/validators/quality_score.py
index 819c5057..2e7efa04 100644
--- a/src/skillevaluator/validators/quality_score.py
+++ b/src/skillevaluator/validators/quality_score.py
@@ -42,6 +42,12 @@ logger = get_logger(__name__)
_WORD_CHAR = r"A-Za-z0-9_"
_MARKDOWN_LINK_START_RE = re.compile(r"\[[^\]\n]*\]\(\s*")
_MARKDOWN_LINK_CLOSER_RE = re.compile(r"^\s*(?:(?:\"[^\"\n]*\"|'[^'\n]*'|\([^\)\n]*\))\s*)?\)")
+_MARKDOWN_REFERENCE_DEFINITION_RE = re.compile(
+ r"^[ \t]{0,3}\[([^\]\n]+)\]:[ \t]*(?:<([^>\n]*)>|([^ \t\r\n]+))",
+ re.MULTILINE,
+)
+_MARKDOWN_REFERENCE_LINK_RE = re.compile(r"\[([^\]\n]+)\]\[([^\]\n]*)\]")
+_MARKDOWN_SHORTCUT_REFERENCE_RE = re.compile(r"(?<![!\]])\[([^\]\n]+)\](?![\(\[])")
_ERROR_HANDLING_RE = re.compile(
r"\b(?:errors?|exceptions?|invalid|fail(?:s|ed|ure|ures|ing)?|"
r"validat(?:e|es|ed|ing|ion|ions))\b",
@@ -198,8 +204,8 @@ def _without_html_comments(content: str) -> str:
def _markdown_link_targets(content: str) -> list[str]:
- """Extract inline Markdown link targets, including balanced parentheses."""
- content = _without_fenced_code(content)
+ """Extract inline and reference-style Markdown link targets."""
+ content = _without_html_comments(_without_fenced_code(content))
targets = []
skip_until = 0
for match in _MARKDOWN_LINK_START_RE.finditer(content):
@@ -244,6 +250,27 @@ def _markdown_link_targets(content: str) -> list[str]:
targets.append("".join(chars))
elif not closed:
skip_until = max(skip_until, cursor)
+
+ definitions: dict[str, str] = {}
+ definition_label_starts: set[int] = set()
+ for match in _MARKDOWN_REFERENCE_DEFINITION_RE.finditer(content):
+ label = re.sub(r"\s+", " ", match.group(1).strip()).casefold()
+ target = match.group(2) if match.group(2) is not None else match.group(3)
+ definitions.setdefault(label, target)
+ definition_label_starts.add(match.start(1))
+
+ for match in _MARKDOWN_REFERENCE_LINK_RE.finditer(content):
+ label = match.group(2) or match.group(1)
+ normalized = re.sub(r"\s+", " ", label.strip()).casefold()
+ if normalized in definitions:
+ targets.append(definitions[normalized])
+
+ for match in _MARKDOWN_SHORTCUT_REFERENCE_RE.finditer(content):
+ if match.start(1) in definition_label_starts:
+ continue
+ normalized = re.sub(r"\s+", " ", match.group(1).strip()).casefold()
+ if normalized in definitions:
+ targets.append(definitions[normalized])
return targets
@@ -274,9 +301,10 @@ def _is_other_cli_mcp_comparison(sentence: str, mcp_end: int, skill_name: str |
def _mcp_usage_contexts(content: str, skill_name: str | None = None) -> list[str]:
- """Return paragraphs where MCP is used as a capability rather than negated."""
+ """Return sentences where MCP is used as a capability rather than negated."""
content = _without_html_comments(_without_fenced_code(content))
contexts = []
+ seen_contexts = set()
for paragraph in re.split(r"\n\s*\n", content):
for match in _MCP_RE.finditer(paragraph):
sentence_start = max(paragraph.rfind(mark, 0, match.start()) for mark in ".!?") + 1
@@ -294,9 +322,9 @@ def _mcp_usage_contexts(content: str, skill_name: str | None = None) -> list[str
match.end() - sentence_start,
skill_name,
)
- if not is_negated and not is_other_cli_comparison:
- contexts.append(paragraph)
- break
+ if not is_negated and not is_other_cli_comparison and sentence not in seen_contexts:
+ contexts.append(sentence)
+ seen_contexts.add(sentence)
return contexts
diff --git a/tests/validators/test_quality_score.py b/tests/validators/test_quality_score.py
index 016c2395..00dc34e8 100644
--- a/tests/validators/test_quality_score.py
+++ b/tests/validators/test_quality_score.py
@@ -358,6 +358,17 @@ class TestQualityScoreValidator:
assert any("Description contains XML tags" in finding.message for finding in result.findings)
+ def test_self_closing_xml_tag_in_description_remains_quality_error(self, tmp_path: Path):
+ """Self-closing tags use the same XML definition as schema validation."""
+ skill_dir = _write_issue_skill(
+ tmp_path,
+ description="Use when validating an injected <script/> tag.",
+ )
+
+ result = QualityScoreValidator(min_score=0).validate(skill_dir)
+
+ assert any("Description contains XML tags" in finding.message for finding in result.findings)
+
def test_readme_supporting_file_is_allowed(self, quality_skill):
# An unreferenced README.md is a permitted human-facing supporting file
# (SkillEvaluator HOW_TO_CONTRIBUTE_SKILLS.md). The quality_skill SKILL.md does
@@ -390,6 +401,19 @@ class TestQualityScoreValidator:
]
assert readme_findings, "Expected a finding when SKILL.md references README.md"
+ def test_reference_style_readme_link_is_flagged(self, quality_skill: Path):
+ (quality_skill / "README.md").write_text("# Human-facing skill notes\n")
+ skill_md = quality_skill / "SKILL.md"
+ skill_md.write_text(
+ skill_md.read_text() + "\n## More\n\nSee [the overview][readme] for background.\n\n[readme]: README.md\n"
+ )
+
+ result = QualityScoreValidator(min_score=0).validate(quality_skill)
+
+ assert any(
+ "README.md" in finding.message and "references" in finding.message.lower() for finding in result.findings
+ )
+
def test_large_skill_is_high_severity(self, tmp_path):
"""Large skills (>5000 tokens) should produce HIGH severity findings."""
from skillevaluator.models.result import Severity
@@ -658,6 +682,20 @@ class TestQualityScoreKeywordBoundaries:
assert "MCP skill lacks connection/error guidance" in messages
+ def test_unrelated_guidance_in_same_paragraph_does_not_satisfy_mcp_guidance(self, tmp_path: Path):
+ messages = _finding_messages(
+ _write_issue_skill(
+ tmp_path,
+ extra_body=(
+ "\n## Usage\n\n"
+ "- Use the MCP server to enumerate tools.\n"
+ "- Connect to the compilation cache before measuring.\n"
+ ),
+ )
+ )
+
+ assert "MCP skill lacks connection/error guidance" in messages
+
def test_mcp_reconnect_guidance_is_still_accepted(self, tmp_path: Path):
messages = _finding_messages(
_write_issue_skill(
@@ -668,6 +706,20 @@ class TestQualityScoreKeywordBoundaries:
assert "MCP skill lacks connection/error guidance" not in messages
+ def test_later_explicit_mcp_sentence_can_supply_guidance(self, tmp_path: Path):
+ messages = _finding_messages(
+ _write_issue_skill(
+ tmp_path,
+ extra_body=(
+ "\n## Usage\n\n"
+ "Use the MCP server to enumerate tools. "
+ "If the MCP server disconnects, reconnect it.\n"
+ ),
+ )
+ )
+
+ assert "MCP skill lacks connection/error guidance" not in messages
+
def test_mcp_guidance_in_troubleshooting_section_is_accepted(self, tmp_path: Path):
messages = _finding_messages(
_write_issue_skill(
@@ -922,6 +974,11 @@ class TestQualityScoreKeywordBoundaries:
assert quality_score_module._markdown_link_targets(content) == ["advanced.md"]
+ def test_shortcut_reference_style_link_resolves_definition(self):
+ content = "See [readme] for details.\n\n[readme]: README.md\n"
+
+ assert quality_score_module._markdown_link_targets(content) == ["README.md"]
+
@pytest.mark.parametrize(
"target",
[
@@ -987,6 +1044,26 @@ class TestQualityScoreKeywordBoundaries:
assert all("Deeply nested references" not in message for message in messages)
+ def test_commented_markdown_link_is_not_nested(self, tmp_path: Path):
+ skill_dir = _write_issue_skill(tmp_path)
+ references = skill_dir / "references"
+ references.mkdir()
+ (references / "mechanisms.md").write_text("<!-- [example](advanced.md) -->\n")
+
+ messages = _finding_messages(skill_dir)
+
+ assert all("Deeply nested references" not in message for message in messages)
+
+ def test_reference_style_local_markdown_link_is_nested(self, tmp_path: Path):
+ skill_dir = _write_issue_skill(tmp_path)
+ references = skill_dir / "references"
+ references.mkdir()
+ (references / "mechanisms.md").write_text("See [more details][advanced].\n\n[advanced]: advanced.md\n")
+
+ messages = _finding_messages(skill_dir)
+
+ assert "Deeply nested references in mechanisms.md" in messages
+
def test_instruction_action_verbs_do_not_match_inside_words(self, tmp_path: Path):
messages = _finding_messages(
_write_issue_skill(
diff --git a/tests/validators/test_schema.py b/tests/validators/test_schema.py
index 07fe6c7d..78db3ad6 100644
--- a/tests/validators/test_schema.py
+++ b/tests/validators/test_schema.py
@@ -478,6 +478,27 @@ Example.
assert not result.passed
assert any("xml" in err.lower() for err in result.errors)
+ def test_self_closing_xml_tag_in_description_rejected(self, tmp_path: Path):
+ """Self-closing XML tags remain forbidden in frontmatter descriptions."""
+ skill_dir = tmp_path / "self-closing-xml-desc"
+ skill_dir.mkdir()
+ (skill_dir / "SKILL.md").write_text(
+ "---\n"
+ "name: self-closing-xml-desc\n"
+ 'description: "Use when validating an injected <tool/> tag."\n'
+ "metadata:\n"
+ " author: Test User <[email protected]>\n"
+ "---\n\n"
+ "# Self-closing XML Description\n\n"
+ "## Instructions\n\n1. Run.\n\n"
+ "## Examples\n\nExample.\n"
+ )
+
+ result = SchemaValidator().validate(skill_dir)
+
+ assert not result.passed
+ assert any("xml" in err.lower() for err in result.errors)
+
def test_comparison_operators_in_description_allowed(self, tmp_path: Path):
"""Test threshold comparators in descriptions are not treated as XML tags."""
skill_dir = tmp_path / "threshold-monitor"