Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(quality): avoid substring false positives
Christopher Kevinbb485658839f
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex e6c80918..43edeac7 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -74,6 +74,10 @@ All notable changes to SkillEvaluator are documented in this file.### Fixed+- Quality scoring now uses boundary-aware and context-aware matching for XML tags,+ reserved names, MCP guidance, README references, time references, exclusivity+ language, instruction action verbs, and nested Markdown links, avoiding+ incidental-word score changes.- Public benchmark cards now omit policy profiles, redact absolute host paths,and normalize imported internal or retired metadata before publication.- Previous-version validation now rejects catalog-wide scalar reuse and removaldiff --git a/src/skillevaluator/models/skill.py b/src/skillevaluator/models/skill.pyindex 341eda2a..f780a1ad 100644--- a/src/skillevaluator/models/skill.py+++ b/src/skillevaluator/models/skill.py@@ -23,7 +23,7 @@ from skillevaluator.constants import (RESERVED_SKILL_NAMES,)-_XML_TAG_RE = re.compile(r"<[a-zA-Z][^>]*>")+XML_TAG_RE = re.compile(r"</?[A-Za-z][A-Za-z0-9_-]*(?:\s[^<>]*)?>|</?[A-Za-z][A-Za-z0-9_-]*\s[^<>]*$")#: Strict, bounded ASCII semantic-version pattern for optional metadata.version labels._SEMVER_COMPONENT = r"(?:0|[1-9][0-9]{0,8})"@@ -150,10 +150,10 @@ class SkillFrontmatter(BaseModel):if v.endswith("-"):raise ValueError(f"Skill name '{v}' cannot end with a hyphen")- if any(w in v.lower() for w in RESERVED_SKILL_NAMES):+ if any(segment in RESERVED_SKILL_NAMES for segment in v.lower().split("-")):raise ValueError(f"Skill name '{v}' must not contain reserved words {RESERVED_SKILL_NAMES}")- if _XML_TAG_RE.search(v):+ if XML_TAG_RE.search(v):raise ValueError(f"Skill name '{v}' must not contain XML tags")return v@@ -162,7 +162,7 @@ class SkillFrontmatter(BaseModel):@classmethoddef validate_description_content(cls, v: str) -> str:"""Reject descriptions that contain XML tags."""- if _XML_TAG_RE.search(v):+ if XML_TAG_RE.search(v):raise ValueError("Skill description must not contain XML tags")return vdiff --git a/src/skillevaluator/validators/quality_score.py b/src/skillevaluator/validators/quality_score.pyindex 7f947bbb..819c5057 100644--- a/src/skillevaluator/validators/quality_score.py+++ b/src/skillevaluator/validators/quality_score.py@@ -18,7 +18,9 @@ gap analysis (H1-H4, M1-M5, L1, L4), OpenAI Skill Evals.from __future__ import annotations+import posixpathimport re+from collections.abc import Iterablefrom pathlib import Pathimport yaml@@ -32,10 +34,340 @@ from skillevaluator.constants import (from skillevaluator.logging_config import get_loggerfrom skillevaluator.models.quality import QualityScoreResultfrom skillevaluator.models.result import Finding, Severity, ValidationResult+from skillevaluator.models.skill import XML_TAG_REfrom skillevaluator.validators.base import ValidatorBaselogger = get_logger(__name__)+_WORD_CHAR = r"A-Za-z0-9_"+_MARKDOWN_LINK_START_RE = re.compile(r"\[[^\]\n]*\]\(\s*")+_MARKDOWN_LINK_CLOSER_RE = re.compile(r"^\s*(?:(?:\"[^\"\n]*\"|'[^'\n]*'|\([^\)\n]*\))\s*)?\)")+_ERROR_HANDLING_RE = re.compile(+ r"\b(?:errors?|exceptions?|invalid|fail(?:s|ed|ure|ures|ing)?|"+ r"validat(?:e|es|ed|ing|ion|ions))\b",+ re.IGNORECASE,+)+_MCP_RE = re.compile(r"\bmcp\b", re.IGNORECASE)+_NEGATED_MCP_RES = (+ re.compile(+ r"\b(?:(?:does|do|did|should|must|shall|can|could|would)\s+not|"+ r"(?:doesn't|don't|didn't|shouldn't|mustn't|shan't|can't|couldn't|wouldn't)|never)\s+"+ r"(?:\w+\s+){0,3}mcp\b",+ re.IGNORECASE,+ ),+ re.compile(r"\bwithout\s+(?:an?\s+)?mcp\b", re.IGNORECASE),+ re.compile(r"\b(?:no|not\s+(?:an?\s+)?)mcp\b", re.IGNORECASE),+ re.compile(+ r"\bmcp\b\s+(?:"+ r"(?:(?:is|are|was|were)\s+not|(?:isn't|aren't|wasn't|weren't))|"+ r"(?:(?:should|must|shall|can|could|would)\s+not|"+ r"(?:shouldn't|mustn't|shan't|can't|couldn't|wouldn't))\s+be"+ r")\s+"+ r"(?:used|required|needed|enabled|supported|involved)\b",+ re.IGNORECASE,+ ),+ re.compile(+ r"\b(?:avoid(?:s|ed|ing)?|disabl(?:e|es|ed|ing)|exclud(?:e|es|ed|ing))\s+"+ r"(?:(?:using|relying\s+on|depending\s+on)\s+)?(?:an?\s+|the\s+)?mcp\b"+ r"(?!\s+(?:errors?|failures?|timeouts?|issues?|problems?|disconnects?|outages?))",+ re.IGNORECASE,+ ),+ re.compile(+ r"\bmcp\b(?:\s+(?:support|integration|access|capability|server))?\s+"+ r"(?:is|are|was|were|remains?|stays?)\s+"+ r"(?:disabled|excluded|unavailable|unsupported)\b",+ re.IGNORECASE,+ ),+ re.compile(r"\bmcp[-\s]+free\b", re.IGNORECASE),+)+_MCP_GUIDANCE_RES = (+ re.compile(r"\bconnect(?:s|ed|ing|ion|ions)?\b", re.IGNORECASE),+ re.compile(r"\breconnect(?:s|ed|ing|ion|ions)?\b", re.IGNORECASE),+ re.compile(r"\bretr(?:y|ies|ied|ying)\b", re.IGNORECASE),+ re.compile(r"\btimeouts?\b", re.IGNORECASE),+ re.compile(r"\bserver\b[^\n.!?]{0,80}\brunning\b", re.IGNORECASE),+ re.compile(r"\bapi\b[^\n.!?]{0,40}\bkeys?\b", re.IGNORECASE),+)+_MARKDOWN_H2_RE = re.compile(r"^##[ \t]+(.+?)[ \t]*\r?$", re.MULTILINE)+_MCP_SUPPORT_HEADINGS = frozenset({"troubleshooting", "common issues", "faq"})+_MCP_SUPPORT_SUBJECT_RE = re.compile(r"\b(?:connections?|sessions?|api\s+keys?)\b", re.IGNORECASE)+_TIME_REFERENCE_RE = re.compile(+ r"\b(?:before|after|as of|until)\s+(?:the\s+year\s+)?(?:19\d{2}|2\d{3})\b",+ re.IGNORECASE,+)+_NON_TEMPORAL_COUNT_RE = re.compile(+ r"^\s+(?:iterations?|tokens?|bytes?|kilobytes?|megabytes?|gigabytes?|"+ r"milliseconds?|seconds?|minutes?|hours?|rows?|items?|attempts?|samples?|steps?|calls?)\b",+ re.IGNORECASE,+)+_EXCLUSIVITY_RE = re.compile(+ r"\breplaces\s+all\s+"+ r"(?P<modifiers>(?:(?:[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\s+){0,6})"+ r"(?:tools|skills|alternatives|solutions|approaches)\b",+ re.IGNORECASE,+)+_NON_EXCLUSIVE_REPLACEMENT_MODIFIERS = frozenset(+ {"deprecated", "legacy", "obsolete", "old", "removed", "retired", "superseded"}+)+++def _contains_term(text: str, term: str) -> bool:+ """Match a word or phrase without accepting it inside another word."""+ normalized = term.strip()+ return bool(+ re.search(+ rf"(?<![{_WORD_CHAR}]){re.escape(normalized)}(?![{_WORD_CHAR}])",+ text,+ re.IGNORECASE,+ )+ )+++def _contains_any_term(text: str, terms: Iterable[str]) -> bool:+ return any(_contains_term(text, term) for term in terms)+++def _has_api_documentation(content: str) -> bool:+ api_patterns = (+ r"\bimport\s+[A-Za-z_][A-Za-z0-9_.]*",+ r"\bfrom\s+[A-Za-z_][A-Za-z0-9_.]*\s+import\b",+ r"\bapi\b",+ r"\bmodules?\b",+ r"\blibrar(?:y|ies)\b",+ r"\bpackages?\b",+ r"\bclasses?\b",+ r"\bfunctions?\b",+ )+ return any(re.search(pattern, content, re.IGNORECASE) for pattern in api_patterns)+++def _without_fenced_code(content: str) -> str:+ """Mask fenced Markdown code while preserving line boundaries."""+ if "```" not in content and "~~~" not in content:+ return content++ visible = []+ fence_char = ""+ fence_length = 0+ for line in content.splitlines(keepends=True):+ body = line.rstrip("\r\n")+ line_ending = line[len(body) :]+ leading_spaces = len(body) - len(body.lstrip(" "))+ marker = None+ if leading_spaces <= 3:+ stripped = body[leading_spaces:]+ if stripped.startswith(("`", "~")):+ char = stripped[0]+ run_length = len(stripped) - len(stripped.lstrip(char))+ remainder = stripped[run_length:]+ if run_length >= 3 and (char != "`" or "`" not in remainder):+ marker = (char, run_length, remainder)++ if not fence_char:+ if marker is None:+ visible.append(line)+ continue+ fence_char, fence_length, _ = marker+ elif marker is not None:+ char, run_length, remainder = marker+ if char == fence_char and run_length >= fence_length and not remainder.strip():+ fence_char = ""+ fence_length = 0+ visible.append(line_ending)+ return "".join(visible)+++def _without_html_comments(content: str) -> str:+ """Mask HTML comments while preserving line boundaries."""+ if "<!--" not in content:+ return content++ visible = []+ cursor = 0+ while True:+ comment_start = content.find("<!--", cursor)+ if comment_start == -1:+ visible.append(content[cursor:])+ break+ visible.append(content[cursor:comment_start])+ closing_marker = content.find("-->", comment_start + 4)+ comment_end = len(content) if closing_marker == -1 else closing_marker + 3+ visible.append("".join(char if char in "\r\n" else " " for char in content[comment_start:comment_end]))+ cursor = comment_end+ return "".join(visible)+++def _markdown_link_targets(content: str) -> list[str]:+ """Extract inline Markdown link targets, including balanced parentheses."""+ content = _without_fenced_code(content)+ targets = []+ skip_until = 0+ for match in _MARKDOWN_LINK_START_RE.finditer(content):+ if match.start() < skip_until:+ continue+ cursor = match.end()+ if cursor >= len(content):+ continue+ if content[cursor] == "<":+ end = content.find(">", cursor + 1)+ if end != -1 and _MARKDOWN_LINK_CLOSER_RE.match(content[end + 1 :]):+ targets.append(content[cursor + 1 : end])+ elif end == -1:+ skip_until = len(content)+ else:+ skip_until = end + 1+ continue++ chars = []+ depth = 0+ closed = False+ while cursor < len(content):+ char = content[cursor]+ if char == "\\" and cursor + 1 < len(content):+ chars.append(content[cursor + 1])+ cursor += 2+ continue+ if char == "(":+ depth += 1+ elif char == ")":+ if depth == 0:+ closed = True+ break+ depth -= 1+ elif char.isspace():+ if depth == 0:+ closed = _MARKDOWN_LINK_CLOSER_RE.match(content[cursor:]) is not None+ break+ chars.append(char)+ cursor += 1+ if chars and depth == 0 and closed:+ targets.append("".join(chars))+ elif not closed:+ skip_until = max(skip_until, cursor)+ return targets+++def _is_other_cli_mcp_comparison(sentence: str, mcp_end: int, skill_name: str | None) -> bool:+ """Return whether a sentence contrasts other CLIs' MCP use with this skill."""+ if not skill_name:+ return False++ before_mcp = sentence[:mcp_end]+ if not re.search(+ r"\b[a-z0-9]+(?:-[a-z0-9]+)*-cli\b[^.!?]{0,80}\buses?\b[^.!?]{0,80}\bmcp\b",+ before_mcp,+ re.IGNORECASE,+ ):+ return False++ after_mcp = sentence[mcp_end:]+ if re.search(r"\bnot\s+(?:this|the\s+current)\s+skill\b", after_mcp, re.IGNORECASE):+ return True++ name_parts = [part for part in re.split(r"[-_\s]+", skill_name) if part]+ aliases = (" ".join(name_parts[index:]) for index in range(max(1, len(name_parts) - 1)))+ return any(+ re.search(rf"\bnot\s+{re.escape(alias)}\b", after_mcp, re.IGNORECASE)+ for alias in aliases+ if len(alias.split()) >= 2+ )+++def _mcp_usage_contexts(content: str, skill_name: str | None = None) -> list[str]:+ """Return paragraphs where MCP is used as a capability rather than negated."""+ content = _without_html_comments(_without_fenced_code(content))+ contexts = []+ for paragraph in re.split(r"\n\s*\n", content):+ for match in _MCP_RE.finditer(paragraph):+ sentence_start = max(paragraph.rfind(mark, 0, match.start()) for mark in ".!?") + 1+ sentence_ends = [paragraph.find(mark, match.end()) for mark in ".!?"]+ sentence_end = min((end for end in sentence_ends if end != -1), default=len(paragraph))+ sentence = paragraph[sentence_start:sentence_end]+ relative_mcp_start = match.start() - sentence_start+ is_negated = any(+ negated.start() <= relative_mcp_start < negated.end()+ for pattern in _NEGATED_MCP_RES+ for negated in pattern.finditer(sentence)+ )+ is_other_cli_comparison = _is_other_cli_mcp_comparison(+ sentence,+ match.end() - sentence_start,+ skill_name,+ )+ if not is_negated and not is_other_cli_comparison:+ contexts.append(paragraph)+ break+ return contexts+++def _markdown_h2_sections(content: str) -> Iterable[tuple[str, str]]:+ """Yield H2 headings and bodies without a backtracking multi-line pattern."""+ content = _without_html_comments(_without_fenced_code(content))+ matches = list(_MARKDOWN_H2_RE.finditer(content))+ for index, match in enumerate(matches):+ body_start = match.end()+ if body_start < len(content) and content[body_start] == "\n":+ body_start += 1+ body_end = matches[index + 1].start() if index + 1 < len(matches) else len(content)+ yield match.group(1).strip(), content[body_start:body_end]+++def _has_mcp_guidance(content: str, usage_contexts: list[str]) -> bool:+ """Return whether usage or a clearly MCP-related support section has guidance."""+ if any(pattern.search(context) for context in usage_contexts for pattern in _MCP_GUIDANCE_RES):+ return True+ previous_heading = ""+ previous_section = ""+ for heading, section in _markdown_h2_sections(content):+ normalized_heading = heading.casefold()+ heading_mentions_mcp = bool(_MCP_RE.search(heading))+ is_support_section = normalized_heading in _MCP_SUPPORT_HEADINGS or (+ heading_mentions_mcp and any(label in normalized_heading for label in _MCP_SUPPORT_HEADINGS)+ )+ has_guidance = any(pattern.search(section) for pattern in _MCP_GUIDANCE_RES)+ explicitly_mcp_related = heading_mentions_mcp or bool(_MCP_RE.search(section))+ follows_mcp_section = bool(+ _MCP_RE.search(previous_heading)+ and _mcp_usage_contexts(previous_section)+ and _MCP_SUPPORT_SUBJECT_RE.search(section)+ )+ if is_support_section and has_guidance and (explicitly_mcp_related or follows_mcp_section):+ return True+ previous_heading = heading+ previous_section = section+ return False+++def _has_exclusive_replacement(content: str) -> bool:+ """Return whether content claims to replace a category of composable tooling."""+ for match in _EXCLUSIVITY_RE.finditer(content):+ modifiers = set(re.findall(r"[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*", match.group("modifiers").lower()))+ if not modifiers.intersection(_NON_EXCLUSIVE_REPLACEMENT_MODIFIERS):+ return True+ return False+++def _has_time_reference(content: str) -> bool:+ for match in _TIME_REFERENCE_RE.finditer(content):+ if not _NON_TEMPORAL_COUNT_RE.match(content[match.end() : match.end() + 32]):+ return True+ return False+++def _has_nested_markdown_reference(content: str) -> bool:+ """Return whether a reference document links to another local Markdown document."""+ for target in _markdown_link_targets(content):+ path = re.split(r"[?#]", target, maxsplit=1)[0].replace("\\", "/")+ lowered = path.lower()+ if not lowered.endswith(".md"):+ continue+ if re.match(r"^[a-z][a-z0-9+.-]*:", target, re.IGNORECASE) or path.startswith(("//", "/")):+ continue+ if posixpath.normpath(path).lower() == "../skill.md":+ continue+ return True+ return False+class QualityScoreValidator(ValidatorBase):"""Evaluates skill quality across 4 weighted dimensions.@@ -306,11 +638,44 @@ class QualityScoreValidator(ValidatorBase):def _references_readme(content: str) -> bool:"""Return True if SKILL.md points agents at a README.md.- A markdown link (``[text](README.md)``), an inline-code path- (```` `README.md` ````), or a bare path mention all count, since any of- them can cause an agent to load the README under progressive disclosure.+ Markdown links and explicit instructions to read/open/load the file count.+ Merely naming README.md, including negative guidance not to load it, does+ not pull the file into agent context."""- return bool(re.search(r"README\.md", content, re.IGNORECASE))+ content = _without_html_comments(_without_fenced_code(content))+ for link_target in _markdown_link_targets(content):+ target = re.split(r"[?#]", link_target, maxsplit=1)[0].replace("\\", "/")+ if posixpath.basename(posixpath.normpath(target)).lower() == "readme.md":+ return True++ action_re = re.compile(+ r"\b(?:read|open|load|consult|review|see|use|follow|refer\s+to)\b"+ r"[^\n.!?]{0,40}\breadme\.md\b",+ re.IGNORECASE,+ )+ negated_action_re = re.compile(+ r"\b(?:(?:(?:do|does|did|should|must|shall|can|could|would)\s+not|"+ r"(?:don't|doesn't|didn't|shouldn't|mustn't|shan't|can't|couldn't|wouldn't))"+ r"(?:\s+need\s+to)?|"+ r"(?:is|are|was|were)\s+not\s+(?:allowed|permitted|required|expected)\s+to|"+ r"cannot|never|not\s+to|need\s+not|no\s+need\s+to)\s+"+ r"(?:(?:ever|directly|automatically|accidentally|normally)\s+){0,2}"+ r"(?:read|open|load|consult|review|see|use|follow|refer\s+to)\b"+ r"[^\n.!?]{0,40}?\breadme\.md\b",+ re.IGNORECASE,+ )+ passive_action_re = re.compile(+ r"\breadme\.md\b[^\n.!?]{0,40}"+ r"\b(?:should|must|needs?\s+to|is\s+required\s+to)\s+be\s+"+ r"(?:read|opened|loaded|consulted|reviewed|followed|used)\b",+ re.IGNORECASE,+ )+ normalized_content = re.sub(r"\s+", " ", content)+ for sentence in re.split(r"(?<=[.!?])\s+", normalized_content):+ affirmative_content = negated_action_re.sub(" ", sentence)+ if action_re.search(affirmative_content) or passive_action_re.search(affirmative_content):+ return True+ return Falsedef _check_frontmatter_correctness(self,@@ -320,11 +685,10 @@ class QualityScoreValidator(ValidatorBase):) -> None:"""Validate frontmatter fields that go beyond basic SchemaValidator checks."""# XML tags in non-name/description fields (Anthropic H3)- xml_tag_re = re.compile(r"</?[a-zA-Z][a-zA-Z0-9_-]*[\s>]")for key, val in fm.items():if key in ("name", "description"):continue- if xml_tag_re.search(str(val)):+ if XML_TAG_RE.search(str(val)):dim.deduct(15,"error",@@ -344,17 +708,17 @@ class QualityScoreValidator(ValidatorBase):f"Invalid name format: '{name}' (lowercase/numbers/hyphens only)","Use only lowercase letters, numbers, and hyphens",)- if any(w in name.lower() for w in QUALITY_RESERVED_NAMES):+ if _contains_any_term(name, QUALITY_RESERVED_NAMES):dim.deduct(15,"error","Name contains reserved word (anthropic, claude)","Remove reserved words from skill name",)- if "<" in name or ">" in name:+ if XML_TAG_RE.search(name):dim.deduct(15, "error", "Name contains XML tags", "Remove XML tags from skill name")- if desc and ("<" in desc or ">" in desc):+ if desc and XML_TAG_RE.search(desc):dim.deduct(15, "error", "Description contains XML tags", "Remove XML tags from description")def _check_type_specific(@@ -409,17 +773,7 @@ class QualityScoreValidator(ValidatorBase):"Lib-based skill missing pyproject.toml","Add pyproject.toml with package metadata and dependencies",)- api_kw = [- "import",- "from ",- "api",- "module",- "library",- "package",- "class ",- "function",- ]- if not any(kw in content.lower() for kw in api_kw):+ if not _has_api_documentation(content):dim.deduct(10,"warning",@@ -437,7 +791,7 @@ class QualityScoreValidator(ValidatorBase):present_res = [d for d in QUALITY_RESOURCE_DIRS if (skill_path / d).exists()]if present_res:res_kw = ["template", "asset", "design", "style", "css", "html", "resource"]- if not any(kw in content.lower() for kw in res_kw):+ if not _contains_any_term(content, res_kw):dim.deduct(5,"info",@@ -447,7 +801,7 @@ class QualityScoreValidator(ValidatorBase):elif skill_type == "resource-based":res_kw = ["template", "asset", "design", "style", "css", "html", "resource"]- if not any(kw in content.lower() for kw in res_kw):+ if not _contains_any_term(content, res_kw):dim.deduct(10,"warning",@@ -498,7 +852,7 @@ class QualityScoreValidator(ValidatorBase):)trigger_words = ["use", "when", "for", "helps", "allows"]- if not any(w in desc.lower() for w in trigger_words):+ if not _contains_any_term(desc, trigger_words):dim.deduct(10,"info",@@ -507,7 +861,7 @@ class QualityScoreValidator(ValidatorBase):)vague_words = ["something", "things", "stuff", "various", "general"]- if any(w in desc.lower() for w in vague_words):+ if _contains_any_term(desc, vague_words):dim.deduct(15,"warning",@@ -516,7 +870,7 @@ class QualityScoreValidator(ValidatorBase):)person_phrases = ["i can", "i will", "you can", "you should", "your", "my", "we can"]- if any(p in desc.lower() for p in person_phrases):+ if _contains_any_term(desc, person_phrases):dim.deduct(15,"warning",@@ -527,11 +881,7 @@ class QualityScoreValidator(ValidatorBase):# Broad description without negative triggers (M1)generic = ["data", "files", "documents", "project", "manage", "handle", "process"]negatives = ["not for", "do not use", "instead use", "except when", "not when"]- if (- len(desc) > 100- and any(t in desc.lower() for t in generic)- and not any(n in desc.lower() for n in negatives)- ):+ if len(desc) > 100 and _contains_any_term(desc, generic) and not _contains_any_term(desc, negatives):dim.deduct(5,"info",@@ -545,11 +895,8 @@ class QualityScoreValidator(ValidatorBase):"the only way to","do not use any other","this skill handles everything",- "replaces all other",- "replaces all",]- cl = content.lower()- if any(p in cl for p in exclusivity):+ if _contains_any_term(content, exclusivity) or _has_exclusive_replacement(content):dim.deduct(5,"info",@@ -588,8 +935,7 @@ class QualityScoreValidator(ValidatorBase):) -> None:dim = qs.reliability- err_kw = ["error", "exception", "invalid", "fail", "validation", "check"]- if any(kw in content.lower() for kw in err_kw):+ if _ERROR_HANDLING_RE.search(content):qs.has_error_handling = Trueelse:dim.deduct(@@ -629,15 +975,14 @@ class QualityScoreValidator(ValidatorBase):)# MCP connection guidance (M2)- if re.search(r"\bmcp\b", content, re.IGNORECASE):- conn_kw = ["connect", "reconnect", "retry", "timeout", "server.*running", "api.*key"]- if not any(re.search(kw, content, re.IGNORECASE) for kw in conn_kw):- dim.deduct(- 10,- "warning",- "MCP skill lacks connection/error guidance",- "Add MCP troubleshooting: connection verification, retry logic",- )+ mcp_contexts = _mcp_usage_contexts(content, qs.skill_name)+ if mcp_contexts and not _has_mcp_guidance(content, mcp_contexts):+ dim.deduct(+ 10,+ "warning",+ "MCP skill lacks connection/error guidance",+ "Add MCP troubleshooting: connection verification, retry logic",+ )def _check_script_reliability(self, dim, skill_path: Path) -> None:scripts_dir = skill_path / "scripts"@@ -729,7 +1074,8 @@ class QualityScoreValidator(ValidatorBase):f"large skill bodies increase token cost after invocation; long or unfocused "f"top-level descriptions can degrade agent routing accuracy"),- "Move examples, reference material, and detailed docs to the references/ directory",+ "Keep required sections concise; move detailed examples, reference material, "+ "and supporting docs to the references/ directory",)# Repetition (compare against non-empty lines to avoid false positives from blank lines)@@ -753,8 +1099,8 @@ class QualityScoreValidator(ValidatorBase):else:section = ""if section:- action_words = ["use", "call", "run", "execute", "pass", "set"]- if not any(w in section.lower() for w in action_words):+ action_words = ["use", "call", "run", "execute", "pass", "set", "add", "mark", "update", "keep"]+ if not _contains_any_term(section, action_words):dim.deduct(15,"warning",@@ -771,7 +1117,7 @@ class QualityScoreValidator(ValidatorBase):# Corporate buzzwordscomplex_words = ["utilize", "facilitate", "leverage", "paradigm", "synergy"]- if any(w in content.lower() for w in complex_words):+ if _contains_any_term(content, complex_words):dim.deduct(5,"info",@@ -780,16 +1126,13 @@ class QualityScoreValidator(ValidatorBase):)# Time-sensitive info- time_pats = [r"before \d{4}", r"after \d{4}", r"as of \d{4}", r"until \d{4}"]- for pat in time_pats:- if re.search(pat, content, re.IGNORECASE):- dim.deduct(- 5,- "info",- "Time-sensitive information detected",- "Avoid dates that become outdated; use 'old patterns' section",- )- break+ if _has_time_reference(content):+ dim.deduct(+ 5,+ "info",+ "Time-sensitive information detected",+ "Avoid dates that become outdated; use 'old patterns' section",+ )# Reference file namingrefs_dir = skill_path / "references"@@ -810,7 +1153,7 @@ class QualityScoreValidator(ValidatorBase):for ref in refs_dir.glob("*.md"):try:rc = ref.read_text(encoding="utf-8")- if re.findall(r"\[[^\]]*\]\([^)]*\.md\)", rc):+ if _has_nested_markdown_reference(rc):dim.deduct(10,"warning",diff --git a/tests/validators/test_quality_score.py b/tests/validators/test_quality_score.pyindex d2352464..016c2395 100644--- a/tests/validators/test_quality_score.py+++ b/tests/validators/test_quality_score.py@@ -7,6 +7,7 @@ from pathlib import Pathimport pytest+import skillevaluator.validators.quality_score as quality_score_modulefrom skillevaluator.models.quality import QualityScoreResult, score_to_gradefrom skillevaluator.validators.quality_score import QualityScoreValidator@@ -91,6 +92,46 @@ def bad_skill(tmp_path: Path) -> Path:return skill_dir+def _write_issue_skill(+ tmp_path: Path,+ *,+ name: str = "compile-time-probe",+ description: str = "Use when mapping kernels to modules and finding redundant builds.",+ instructions: str = "- Run the probe.\n- Read the report.",+ troubleshooting: str = "Read diagnostic output when the probe reports an error.",+ extra_body: str = "",+) -> Path:+ """Write a complete guide skill for issue #30 scoring regressions."""+ skill_dir = tmp_path / name+ skill_dir.mkdir()+ (skill_dir / "SKILL.md").write_text(+ "---\n"+ f"name: {name}\n"+ f"description: {description}\n"+ "version: 0.1.0\n"+ "metadata:\n"+ " author: Example\n"+ " tags:\n"+ " - performance\n"+ "---\n\n"+ "## Purpose\n\nMeasure compile time.\n\n"+ "## Instructions\n\n"+ f"{instructions}\n\n"+ "## Examples\n\n```bash\nprobe measure -- python app.py\n```\n\n"+ "## Prerequisites\n\nNone.\n\n"+ "## Limitations\n\nNone known.\n\n"+ "## Troubleshooting\n\n"+ f"{troubleshooting}\n"+ f"{extra_body}"+ )+ return skill_dir+++def _finding_messages(skill_dir: Path) -> list[str]:+ result = QualityScoreValidator(min_score=0).validate(skill_dir)+ return [finding.message for finding in result.findings]++class TestScoreToGrade:def test_grade_a(self):assert score_to_grade(95.0) == "A"@@ -297,6 +338,26 @@ class TestQualityScoreValidator:assert any("Description contains XML tags" in finding.message for finding in result.findings)+ def test_closing_xml_tag_in_description_remains_quality_error(self, tmp_path):+ """Closing tags use the same XML definition as schema validation."""+ skill_dir = tmp_path / "closing-xml-desc"+ skill_dir.mkdir()+ (skill_dir / "SKILL.md").write_text(+ "---\n"+ "name: closing-xml-desc\n"+ 'description: "Use when validating a closing </tool> tag."\n'+ "metadata:\n"+ " author: Test User <[email protected]>\n"+ "---\n\n"+ "# Closing XML Description\n\n"+ "## Instructions\n\n1. Inspect frontmatter quality findings.\n\n"+ "## Examples\n\nValidate the skill.\n"+ )++ result = QualityScoreValidator(min_score=0).validate(skill_dir)++ assert any("Description contains XML tags" in finding.message for finding in result.findings)+def test_readme_supporting_file_is_allowed(self, quality_skill):# An unreferenced README.md is a permitted human-facing supporting file# (SkillEvaluator HOW_TO_CONTRIBUTE_SKILLS.md). The quality_skill SKILL.md does@@ -355,6 +416,8 @@ class TestQualityScoreValidator:assert large_findings[0].severity == Severity.HIGHassert "recommended max <5000" in large_findings[0].messageassert "long or unfocused top-level descriptions" in large_findings[0].message+ assert "Keep required sections concise" in large_findings[0].suggestion+ assert "references/" in large_findings[0].suggestiondef test_above_6000_tokens_uses_same_5000_recommendation(self, tmp_path):"""Skills above 6000 tokens should still use the single >5000 recommendation."""@@ -390,3 +453,557 @@ class TestQualityScoreValidator:v = QualityScoreValidator()assert "Quality" in v.nameassert "Correctness" in v.description+++class TestQualityScoreKeywordBoundaries:+ @pytest.mark.parametrize(+ "description",+ [+ "Use when mapping kernel -> module relationships during builds.",+ "Use when cutting rebuild cost by >30% for large modules.",+ "Use when targeting compile steps that should finish in <1s.",+ ],+ )+ def test_comparison_syntax_is_not_reported_as_xml(self, tmp_path: Path, description: str):+ messages = _finding_messages(_write_issue_skill(tmp_path, description=description))++ assert "Description contains XML tags" not in messages++ def test_reserved_names_match_complete_name_segments(self, tmp_path: Path):+ messages = _finding_messages(_write_issue_skill(tmp_path, name="philanthropic-grant-matcher"))++ assert "Name contains reserved word (anthropic, claude)" not in messages++ def test_reserved_name_segment_is_still_reported(self, tmp_path: Path):+ messages = _finding_messages(_write_issue_skill(tmp_path, name="anthropic-grant-matcher"))++ assert "Name contains reserved word (anthropic, claude)" in messages++ def test_person_phrases_do_not_match_inside_words(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(+ tmp_path,+ description="Use when generating dummy data fixtures for compile probes.",+ )+ )++ assert "Description uses first/second person" not in messages++ def test_when_to_use_trigger_does_not_match_inside_performance(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(+ tmp_path,+ description="Maps performance characteristics across compilation workloads.",+ )+ )++ assert "Description doesn't mention WHEN to use this skill" in messages++ @pytest.mark.parametrize("incidental_word", ["important", "rapid"])+ def test_lib_documentation_requires_complete_api_terms(self, tmp_path: Path, incidental_word: str):+ skill_dir = _write_issue_skill(+ tmp_path,+ description="Use when analyzing compile speed across build targets.",+ extra_body=f"\n## Notes\n\nThis is {incidental_word} for compile analysis.\n",+ )+ module = skill_dir / "probe"+ module.mkdir()+ (module / "__init__.py").write_text("")+ (skill_dir / "pyproject.toml").write_text("[project]\nname = 'probe'\nversion = '0.1.0'\n")++ messages = _finding_messages(skill_dir)++ assert "Lib-based skill lacks API/import documentation" in messages++ def test_check_by_itself_does_not_claim_error_handling(self, tmp_path: Path):+ skill_dir = _write_issue_skill(+ tmp_path,+ troubleshooting="Check the output directory.",+ )++ result = QualityScoreValidator(min_score=0).validate(skill_dir)++ assert result.metadata["quality_scores"]["metrics"]["has_error_handling"] is False+ assert "No mention of error handling or validation" in [finding.message for finding in result.findings]++ def test_error_terms_still_record_error_handling(self, tmp_path: Path):+ skill_dir = _write_issue_skill(+ tmp_path,+ troubleshooting="If validation fails, report the error and retry the probe.",+ )++ result = QualityScoreValidator(min_score=0).validate(skill_dir)++ assert result.metadata["quality_scores"]["metrics"]["has_error_handling"] is True+ assert "No mention of error handling or validation" not in [finding.message for finding in result.findings]++ def test_negated_mcp_mention_does_not_classify_mcp_skill(self, tmp_path: Path):+ messages = _finding_messages(_write_issue_skill(tmp_path, extra_body="\nThis skill does not use MCP.\n"))++ assert "MCP skill lacks connection/error guidance" not in messages++ def test_postposed_negated_mcp_mention_does_not_classify_mcp_skill(self, tmp_path: Path):+ messages = _finding_messages(_write_issue_skill(tmp_path, extra_body="\nMCP is not used by this skill.\n"))++ assert "MCP skill lacks connection/error guidance" not in messages++ @pytest.mark.parametrize(+ "statement",+ [+ "This skill should not use MCP.",+ "MCP should not be used by this skill.",+ ],+ )+ def test_modal_negated_mcp_mention_does_not_classify_mcp_skill(self, tmp_path: Path, statement: str):+ messages = _finding_messages(_write_issue_skill(tmp_path, extra_body=f"\n{statement}\n"))++ assert "MCP skill lacks connection/error guidance" not in messages++ @pytest.mark.parametrize(+ "statement",+ [+ "This skill avoids MCP.",+ "This skill avoids using MCP.",+ "This workflow excludes the MCP integration.",+ "MCP is disabled for this skill.",+ "MCP support remains unavailable.",+ "This workflow is MCP-free.",+ ],+ )+ def test_negative_mcp_capability_does_not_classify_mcp_skill(self, tmp_path: Path, statement: str):+ messages = _finding_messages(_write_issue_skill(tmp_path, extra_body=f"\n{statement}\n"))++ assert "MCP skill lacks connection/error guidance" not in messages++ def test_avoiding_mcp_failures_remains_mcp_usage(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(tmp_path, extra_body="\nAvoid MCP failures during validation.\n")+ )++ assert "MCP skill lacks connection/error guidance" in messages++ def test_fenced_mcp_example_does_not_classify_mcp_skill(self, tmp_path: Path):+ messages = _finding_messages(_write_issue_skill(tmp_path, extra_body="\n```text\nMCP example output\n```\n"))++ assert "MCP skill lacks connection/error guidance" not in messages++ def test_other_cli_mcp_comparison_does_not_classify_current_skill(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(+ tmp_path,+ name="authenticating-entra-device-code",+ extra_body=(+ "\nNote: sharepoint-cli, gdrive-cli, and glean-cli use MaaS MCP auth "+ "(via nv-discovery-cli), not Entra device code.\n"+ ),+ )+ )++ assert "MCP skill lacks connection/error guidance" not in messages++ def test_current_cli_mcp_comparison_remains_mcp_usage(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(+ tmp_path,+ name="sharepoint-cli",+ extra_body="\nsharepoint-cli uses MaaS MCP auth, not Entra device code.\n",+ )+ )++ assert "MCP skill lacks connection/error guidance" in messages++ def test_mcp_mention_in_html_comment_does_not_classify_skill(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(+ tmp_path,+ extra_body=(+ "\n<!-- Related skills:\n- managing-sharepoint: SharePoint sites and lists (MaaS MCP)\n-->\n"+ ),+ )+ )++ assert "MCP skill lacks connection/error guidance" not in messages++ def test_mcp_guidance_in_html_comment_does_not_satisfy_active_usage(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(+ tmp_path,+ extra_body=(+ "\nUse the MCP server to enumerate tools.\n\n"+ "<!--\n## MCP Troubleshooting\n\n"+ "Reconnect the server and retry after a timeout.\n-->\n"+ ),+ )+ )++ assert "MCP skill lacks connection/error guidance" in messages++ def test_interconnect_does_not_satisfy_mcp_connection_guidance(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(tmp_path, extra_body="\nUse the MCP server for GPU interconnect analysis.\n")+ )++ assert "MCP skill lacks connection/error guidance" in messages++ def test_unrelated_connection_text_does_not_satisfy_mcp_guidance(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(+ tmp_path,+ extra_body=(+ "\nUse the MCP server to enumerate tools.\n\n"+ "## Compilation Cache\n\nConnect to the compilation cache before measuring.\n"+ ),+ )+ )++ assert "MCP skill lacks connection/error guidance" in messages++ def test_mcp_reconnect_guidance_is_still_accepted(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(+ tmp_path,+ extra_body="\nUse the MCP server. Reconnect the server if the session expires.\n",+ )+ )++ assert "MCP skill lacks connection/error guidance" not in messages++ def test_mcp_guidance_in_troubleshooting_section_is_accepted(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(+ tmp_path,+ extra_body=(+ "\n## MCP Usage\n\nUse the MCP server to enumerate tools.\n\n"+ "## Troubleshooting\n\nIf the connection drops, reconnect the server and retry.\n"+ ),+ )+ )++ assert "MCP skill lacks connection/error guidance" not in messages++ def test_mcp_guidance_in_explicitly_titled_support_section_is_accepted(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(+ tmp_path,+ extra_body=(+ "\n## MCP Usage\n\nUse the MCP server to enumerate tools.\n\n"+ "## MCP Troubleshooting\n\nReconnect the server and retry after a timeout.\n"+ ),+ )+ )++ assert "MCP skill lacks connection/error guidance" not in messages++ def test_unrelated_troubleshooting_does_not_satisfy_mcp_guidance(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(+ tmp_path,+ extra_body=(+ "\n## MCP Usage\n\nUse the MCP server to enumerate tools.\n\n"+ "## Troubleshooting\n\nRetry compilation if the local cache is unavailable.\n"+ ),+ )+ )++ assert "MCP skill lacks connection/error guidance" in messages++ def test_unrelated_build_server_troubleshooting_does_not_satisfy_mcp_guidance(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(+ tmp_path,+ extra_body=(+ "\n## MCP Usage\n\nUse the MCP server to enumerate tools.\n\n"+ "## Troubleshooting\n\nIf the build server fails, retry compilation.\n"+ ),+ )+ )++ assert "MCP skill lacks connection/error guidance" in messages++ def test_iteration_count_is_not_time_sensitive_information(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(tmp_path, extra_body="\nUnrolling stops after 2048 iterations.\n")+ )++ assert "Time-sensitive information detected" not in messages++ def test_actual_year_reference_remains_time_sensitive(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(tmp_path, extra_body="\nUse this compatibility path after 2025.\n")+ )++ assert "Time-sensitive information detected" in messages++ def test_future_year_reference_remains_time_sensitive(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(tmp_path, extra_body="\nUse this compatibility path after 2101.\n")+ )++ assert "Time-sensitive information detected" in messages++ def test_replacing_deprecated_calls_is_not_exclusivity(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(tmp_path, extra_body="\nThis release replaces all deprecated launch calls.\n")+ )++ assert "Skill uses exclusivity language that conflicts with composability" not in messages++ @pytest.mark.parametrize(+ "statement",+ [+ "This release replaces all deprecated tool calls.",+ "This migration replaces all legacy build tools.",+ "This migration replaces all other deprecated tool calls.",+ ],+ )+ def test_replacing_obsolete_implementation_details_is_not_exclusivity(self, tmp_path: Path, statement: str):+ messages = _finding_messages(_write_issue_skill(tmp_path, extra_body=f"\n{statement}\n"))++ assert "Skill uses exclusivity language that conflicts with composability" not in messages++ def test_replacing_all_other_tools_remains_exclusivity(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(tmp_path, extra_body="\nThis skill replaces all other build tools.\n")+ )++ assert "Skill uses exclusivity language that conflicts with composability" in messages++ @pytest.mark.parametrize(+ "statement",+ [+ "This skill replaces all tools.",+ "This skill replaces all competing build tools.",+ "This skill replaces all alternative skills.",+ "This skill replaces all competing third-party tools.",+ "This skill replaces all currently available competing build tools.",+ ],+ )+ def test_replacing_all_tooling_remains_exclusivity(self, tmp_path: Path, statement: str):+ messages = _finding_messages(_write_issue_skill(tmp_path, extra_body=f"\n{statement}\n"))++ assert "Skill uses exclusivity language that conflicts with composability" in messages++ def test_readme_negative_guidance_is_not_a_reference(self, tmp_path: Path):+ skill_dir = _write_issue_skill(+ tmp_path,+ extra_body="\nREADME.md is human-facing; do not load it.\n",+ )+ (skill_dir / "README.md").write_text("# Human documentation\n")++ messages = _finding_messages(skill_dir)++ assert all("SKILL.md references README.md" not in message for message in messages)++ def test_readme_load_instruction_is_still_a_reference(self, tmp_path: Path):+ skill_dir = _write_issue_skill(tmp_path, extra_body="\nRead README.md before publishing.\n")+ (skill_dir / "README.md").write_text("# Human documentation\n")++ messages = _finding_messages(skill_dir)++ assert any("SKILL.md references README.md" in message for message in messages)++ def test_wrapped_readme_negative_guidance_is_not_a_reference(self, tmp_path: Path):+ skill_dir = _write_issue_skill(tmp_path, extra_body="\nDo not\nread README.md before publishing.\n")+ (skill_dir / "README.md").write_text("# Human documentation\n")++ messages = _finding_messages(skill_dir)++ assert all("SKILL.md references README.md" not in message for message in messages)++ def test_passive_readme_instruction_is_still_a_reference(self, tmp_path: Path):+ skill_dir = _write_issue_skill(tmp_path, extra_body="\nREADME.md should be read before publishing.\n")+ (skill_dir / "README.md").write_text("# Human documentation\n")++ messages = _finding_messages(skill_dir)++ assert any("SKILL.md references README.md" in message for message in messages)++ def test_passive_negative_readme_guidance_is_not_a_reference(self, tmp_path: Path):+ skill_dir = _write_issue_skill(tmp_path, extra_body="\nREADME.md should not be read by agents.\n")+ (skill_dir / "README.md").write_text("# Human documentation\n")++ messages = _finding_messages(skill_dir)++ assert all("SKILL.md references README.md" not in message for message in messages)++ @pytest.mark.parametrize(+ "statement",+ [+ "Agents should not read README.md.",+ "Agents must not load README.md.",+ "Agents shouldn't consult README.md.",+ "Agents should not ever read README.md.",+ "Agents should not refer to README.md.",+ "Agents don't need to read README.md.",+ "Agents are not required to read README.md.",+ "Agents are not allowed to load README.md.",+ ],+ )+ def test_modal_negative_readme_guidance_is_not_a_reference(self, tmp_path: Path, statement: str):+ skill_dir = _write_issue_skill(tmp_path, extra_body=f"\n{statement}\n")+ (skill_dir / "README.md").write_text("# Human documentation\n")++ messages = _finding_messages(skill_dir)++ assert all("SKILL.md references README.md" not in message for message in messages)++ def test_not_only_readme_instruction_remains_a_reference(self, tmp_path: Path):+ skill_dir = _write_issue_skill(+ tmp_path,+ extra_body="\nAgents should not only read README.md; they should also inspect the release notes.\n",+ )+ (skill_dir / "README.md").write_text("# Human documentation\n")++ messages = _finding_messages(skill_dir)++ assert any("SKILL.md references README.md" in message for message in messages)++ def test_affirmative_readme_instruction_after_negative_one_remains_a_reference(self, tmp_path: Path):+ skill_dir = _write_issue_skill(+ tmp_path,+ extra_body="\nDo not read README.md for setup; review README.md before publishing.\n",+ )+ (skill_dir / "README.md").write_text("# Human documentation\n")++ messages = _finding_messages(skill_dir)++ assert any("SKILL.md references README.md" in message for message in messages)++ def test_markdown_link_to_readme_suffix_is_not_a_readme_reference(self, tmp_path: Path):+ skill_dir = _write_issue_skill(tmp_path, extra_body="\nSee [release notes](NOTREADME.md).\n")+ (skill_dir / "README.md").write_text("# Human documentation\n")+ (skill_dir / "NOTREADME.md").write_text("# Release notes\n")++ messages = _finding_messages(skill_dir)++ assert all("SKILL.md references README.md" not in message for message in messages)++ def test_fenced_readme_link_example_is_not_a_readme_reference(self, tmp_path: Path):+ skill_dir = _write_issue_skill(+ tmp_path,+ extra_body="\n```markdown\nSee [human documentation](README.md).\n```\n",+ )+ (skill_dir / "README.md").write_text("# Human documentation\n")++ messages = _finding_messages(skill_dir)++ assert all("SKILL.md references README.md" not in message for message in messages)++ def test_commented_readme_link_is_not_a_readme_reference(self, tmp_path: Path):+ skill_dir = _write_issue_skill(+ tmp_path,+ extra_body="\n<!-- See [human documentation](README.md). -->\n",+ )+ (skill_dir / "README.md").write_text("# Human documentation\n")++ messages = _finding_messages(skill_dir)++ assert all("SKILL.md references README.md" not in message for message in messages)++ def test_markdown_link_parser_scans_malformed_nested_input_linearly(self):+ read_count = 0++ class CountingText(str):+ __slots__ = ()++ def __getitem__(self, key):+ nonlocal read_count+ if isinstance(key, int):+ read_count += 1+ return super().__getitem__(key)++ content = CountingText("[x](" * 200)++ assert quality_score_module._markdown_link_targets(content) == []+ assert read_count <= len(content)++ def test_inline_backtick_span_at_line_start_does_not_mask_following_links(self):+ content = "```literal```\n[details](advanced.md)\n"++ assert quality_score_module._markdown_link_targets(content) == ["advanced.md"]++ @pytest.mark.parametrize(+ "target",+ [+ "https://agentskills.io/specification.md",+ "../SKILL.md",+ ],+ )+ def test_external_and_parent_markdown_links_are_not_nested_references(self, tmp_path: Path, target: str):+ skill_dir = _write_issue_skill(tmp_path)+ references = skill_dir / "references"+ references.mkdir()+ (references / "mechanisms.md").write_text(f"See [the specification]({target}).\n")++ messages = _finding_messages(skill_dir)++ assert all("Deeply nested references" not in message for message in messages)++ def test_local_markdown_link_remains_a_nested_reference(self, tmp_path: Path):+ skill_dir = _write_issue_skill(tmp_path)+ references = skill_dir / "references"+ references.mkdir()+ (references / "mechanisms.md").write_text("See [more details](advanced.md).\n")++ messages = _finding_messages(skill_dir)++ assert "Deeply nested references in mechanisms.md" in messages++ @pytest.mark.parametrize(+ "target",+ [+ "../other.md",+ "nested/SKILL.md",+ "guide(v2).md",+ ],+ )+ def test_other_local_markdown_paths_remain_nested_references(self, tmp_path: Path, target: str):+ skill_dir = _write_issue_skill(tmp_path)+ references = skill_dir / "references"+ references.mkdir()+ (references / "mechanisms.md").write_text(f"See [more details]({target}).\n")++ messages = _finding_messages(skill_dir)++ assert "Deeply nested references in mechanisms.md" in messages++ def test_external_url_with_balanced_parentheses_is_not_nested(self, tmp_path: Path):+ skill_dir = _write_issue_skill(tmp_path)+ references = skill_dir / "references"+ references.mkdir()+ (references / "mechanisms.md").write_text("See [the specification](https://example.com/spec(v2).md).\n")++ messages = _finding_messages(skill_dir)++ assert all("Deeply nested references" not in message for message in messages)++ def test_fenced_markdown_link_example_is_not_nested(self, tmp_path: Path):+ skill_dir = _write_issue_skill(tmp_path)+ references = skill_dir / "references"+ references.mkdir()+ (references / "mechanisms.md").write_text("```markdown\n[example](advanced.md)\n```\n")++ messages = _finding_messages(skill_dir)++ assert all("Deeply nested references" not in message for message in messages)++ def test_instruction_action_verbs_do_not_match_inside_words(self, tmp_path: Path):+ messages = _finding_messages(+ _write_issue_skill(+ tmp_path,+ instructions="- Because the offset is a runtime property, results vary.",+ )+ )++ assert "Instructions lack clear action verbs" in messages++ @pytest.mark.parametrize("action", ["Add", "Mark", "Update", "Keep"])+ def test_task_list_imperatives_are_clear_action_verbs(self, tmp_path: Path, action: str):+ messages = _finding_messages(+ _write_issue_skill(+ tmp_path,+ instructions=f"- {action} task status as each step completes.",+ )+ )++ assert "Instructions lack clear action verbs" not in messagesdiff --git a/tests/validators/test_schema.py b/tests/validators/test_schema.pyindex 62ce6104..07fe6c7d 100644--- a/tests/validators/test_schema.py+++ b/tests/validators/test_schema.py@@ -354,6 +354,26 @@ Example.assert not result.passed, f"Name containing '{word}' should be rejected"assert any("reserved" in err.lower() for err in result.errors)+ def test_reserved_word_substring_in_name_is_allowed(self, tmp_path: Path):+ """Reserved providers only match complete hyphen-delimited name segments."""+ skill_dir = tmp_path / "philanthropic-grant-matcher"+ skill_dir.mkdir()+ (skill_dir / "SKILL.md").write_text(+ "---\n"+ "name: philanthropic-grant-matcher\n"+ "description: Use when matching philanthropic grants to eligible programs.\n"+ "metadata:\n"+ " author: Test User <[email protected]>\n"+ "---\n\n"+ "# Philanthropic Grant Matcher\n\n"+ "## Instructions\n\n1. Match each grant to an eligible program.\n\n"+ "## Examples\n\nMatch the example grant.\n"+ )++ result = SchemaValidator().validate(skill_dir)++ assert result.passed, result.errors+def test_xml_tags_in_name_rejected(self, tmp_path: Path):"""Test validation fails when name contains XML tags."""skill_dir = tmp_path / "xml-name"@@ -437,6 +457,27 @@ Example.assert not result.passedassert any("xml" in err.lower() or "description" in err.lower() for err in result.errors)+ def test_closing_xml_tag_in_description_rejected(self, tmp_path: Path):+ """Opening and closing XML tags use the same schema and quality definition."""+ skill_dir = tmp_path / "closing-xml-desc"+ skill_dir.mkdir()+ (skill_dir / "SKILL.md").write_text(+ "---\n"+ "name: closing-xml-desc\n"+ 'description: "Use when validating a closing </tool> tag."\n'+ "metadata:\n"+ " author: Test User <[email protected]>\n"+ "---\n\n"+ "# Closing XML Description\n\n"+ "## Instructions\n\n1. Run.\n\n"+ "## Examples\n\nExample.\n"+ )++ result = SchemaValidator().validate(skill_dir)++ assert not result.passed+ assert any("xml" in err.lower() for err in result.errors)+def test_comparison_operators_in_description_allowed(self, tmp_path: Path):"""Test threshold comparators in descriptions are not treated as XML tags."""skill_dir = tmp_path / "threshold-monitor"