Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(hygiene): use CommonMark for dead-link checks (#109)

Rio Yu79d6fd353c9f
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 78f312cf..f1c585ff 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -122,6 +122,32 @@ jobs:
- name: Run tests
run: uv run pytest -q
+ html-parser-compatibility:
+ name: HTML parser compatibility (Python ${{ matrix.python-version }})
+ needs: classify-changes
+ if: ${{ !cancelled() && needs.classify-changes.outputs.docs_only != 'true' }}
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version: ["3.12.11", "3.13.5"]
+ steps:
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ persist-credentials: false
+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+ with:
+ python-version: ${{ matrix.python-version }}
+ - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
+ with:
+ version: "0.11.27"
+ - run: uv sync --extra dev --locked --python "${{ matrix.python-version }}"
+ - name: Run HTML parser compatibility tests
+ run: >-
+ uv run --python "${{ matrix.python-version }}" pytest -q
+ tests/validators/test_markdown.py
+ tests/test_cli_html_parser_compat.py
+
rhel8-security-install:
name: RHEL 8 security install
needs: classify-changes
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2bc6ea7d..bb297579 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,16 @@ All notable changes to SkillEvaluator are documented in this file.
run passed -- agent output travels over the Docker exec API rather than through
the mounts -- and every scored trial then failed with `RewardFileNotFoundError`
while the rewards sat inside the daemon's own filesystem.
+- Dead-link validation now uses the shared CommonMark parser, covering
+ reference-style and HTML links while preserving Markdown image checks and
+ consistently normalizing local destinations. Root-absolute URLs are ignored
+ instead of being treated as host paths; href-only diagnostics collapse
+ repeated links to the same normalized target. Invalid destination bytes do
+ not alias other files, relative URLs that normalize to absolute or
+ drive-relative paths are reported without lookup, lookup failures remain
+ per-link findings, and link diagnostics are bounded and escaped. Malformed
+ frontmatter and repeated unclosed HTML comments no longer abort or stall
+ supporting-document checks.
- Tier 3 accuracy and custom goal judges now retry one malformed (including
empty) or schema-invalid response with a 4096-token output budget before
failing closed, preventing a transient formatting error from making an otherwise
diff --git a/src/skillevaluator/validators/hygiene.py b/src/skillevaluator/validators/hygiene.py
index fb243210..76cea316 100644
--- a/src/skillevaluator/validators/hygiene.py
+++ b/src/skillevaluator/validators/hygiene.py
@@ -13,12 +13,19 @@ from pathlib import Path
from skillevaluator.constants import BANNED_PACKAGES
from skillevaluator.logging_config import get_logger
from skillevaluator.validators.base import ValidationResult, ValidatorBase, iter_scannable_files
+from skillevaluator.validators.markdown import markdown_link_targets, normalized_local_path
logger = get_logger(__name__)
_TEST_FILE_PATTERNS = ("test_*.py", "*_test.py")
+def _link_display(value: str) -> str:
+ """Keep untrusted link diagnostics bounded and on one physical line."""
+ escaped = ascii(value[:160])[1:-1]
+ return escaped[:160] + ("..." if len(value) > 160 or len(escaped) > 160 else "")
+
+
class HygieneValidator(ValidatorBase):
"""Validates code integrity: dead links, dependencies, and test-file presence."""
@@ -70,10 +77,6 @@ class HygieneValidator(ValidatorBase):
message=f"Checking {len(md_files)} markdown files for dead links",
file_count=len(md_files),
)
- link_pattern = re.compile(r"\[([^\]]+)\]\(([^)]+)\)")
- # Pattern to match fenced code blocks (``` or ~~~)
- code_block_pattern = re.compile(r"(```|~~~).*?\1", re.DOTALL)
-
for md_file in md_files:
try:
content = md_file.read_text(encoding="utf-8")
@@ -81,27 +84,31 @@ class HygieneValidator(ValidatorBase):
result.add_warning(f"Could not read {md_file}: {e}")
continue
- # Remove fenced code blocks before checking links
- # This prevents false positives from example links in documentation
- content_without_code = code_block_pattern.sub("", content)
-
- for link_text, link_href in link_pattern.findall(content_without_code):
- # Skip external URLs and anchors
- if link_href.startswith(("http://", "https://", "mailto:", "#")):
+ seen_targets: set[str] = set()
+ seen_invalid: set[str] = set()
+ for link_href in markdown_link_targets(content, include_images=True):
+ try:
+ local_path = normalized_local_path(link_href, allow_directory=True, reject_anchors=True)
+ except ValueError:
+ if link_href in seen_invalid:
+ continue
+ seen_invalid.add(link_href)
+ result.add_error(
+ f"Invalid local link in {_link_display(md_file.name)}: {_link_display(link_href)} "
+ "(absolute or drive-relative path)"
+ )
continue
-
- # Skip links inside inline code (backticks)
- # Check if this link appears in the original content within backticks
- inline_code_check = f"`[{link_text}]({link_href})`"
- if inline_code_check in content:
+ if local_path is None or local_path in seen_targets:
continue
-
- # Resolve and validate relative path
- clean_path = link_href.removeprefix("./").split("#")[0]
- target = md_file.parent / clean_path
-
- if not target.exists():
- result.add_error(f"Dead link in {md_file.name}: [{link_text}]({link_href})")
+ seen_targets.add(local_path)
+ target = md_file.parent / local_path
+
+ try:
+ exists = target.exists()
+ except (OSError, ValueError):
+ exists = False
+ if not exists:
+ result.add_error(f"Dead link in {_link_display(md_file.name)}: {_link_display(link_href)}")
if not result.errors:
result.add_success(
diff --git a/src/skillevaluator/validators/markdown.py b/src/skillevaluator/validators/markdown.py
index fc8769d5..230d4094 100644
--- a/src/skillevaluator/validators/markdown.py
+++ b/src/skillevaluator/validators/markdown.py
@@ -3,13 +3,19 @@
"""Structural Markdown helpers for deterministic validators."""
+import posixpath
import re
from collections.abc import Iterable, Iterator
from html import escape
from html.parser import HTMLParser
+from inspect import signature
+from pathlib import PureWindowsPath
+from urllib.parse import unquote, urlsplit
import yaml
from markdown_it import MarkdownIt
+from markdown_it.rules_inline import html_inline
+from markdown_it.rules_inline.state_inline import StateInline
from markdown_it.token import Token
from skillevaluator.validators.frontmatter_parser import FRONTMATTER_PATTERN
@@ -24,6 +30,65 @@ _SCRIPT_START_RE = re.compile(r"<script(?=[\t\n\r\f />])", re.IGNORECASE | re.AS
_SCRIPT_END_RE = re.compile(r"</script(?=[\t\n\r\f />])", re.IGNORECASE | re.ASCII)
_SCRIPT_ESCAPED_STATES = frozenset({"escaped", "escaped-dash", "escaped-dash-dash"})
_SCRIPT_DOUBLE_ESCAPED_STATES = frozenset({"double-escaped", "double-escaped-dash", "double-escaped-dash-dash"})
+_URL_EDGE_C0_OR_SPACE_RE = re.compile(r"^[\x00-\x20]+|[\x00-\x20]+$")
+_HTML_PARSER_SUPPORTS_ESCAPABLE_CDATA = "escapable" in signature(HTMLParser.set_cdata_mode).parameters
+
+
+def _guarded_html_inline(state: StateInline, silent: bool) -> bool:
+ """Avoid rescanning suffixes for HTML fragments with no possible terminator."""
+ terminator = None
+ if state.src.startswith("<!--", state.pos) and not state.src.startswith(("<!-->", "<!--->"), state.pos):
+ terminator = "-->"
+ elif state.src.startswith("<?", state.pos):
+ terminator = "?>"
+ elif state.src.startswith("<![CDATA[", state.pos):
+ terminator = "]]>"
+ elif state.src.startswith("<!", state.pos) and state.pos + 2 < len(state.src):
+ letter = state.src[state.pos + 2]
+ if "A" <= letter <= "Z" or "a" <= letter <= "z":
+ terminator = ">"
+ if terminator is not None:
+ # Each inline source has its own offsets, even within the same document.
+ if state.env.get("_html_source") is not state.src:
+ state.env["_html_source"] = state.src
+ state.env["_html_ends"] = {end: state.src.rfind(end) for end in ("-->", "?>", "]]>", ">")}
+ if state.pos > state.env["_html_ends"][terminator]:
+ return False
+ return html_inline(state, silent)
+
+
+_MARKDOWN_PARSER.inline.ruler.at("html_inline", _guarded_html_inline)
+
+
+def normalized_local_path(href: str, *, allow_directory: bool = False, reject_anchors: bool = False) -> str | None:
+ """Normalize a local destination, optionally rejecting anchors acquired in normalization.
+
+ Explicit URL schemes and URLs whose raw path begins with ``/`` remain
+ outside local checks. Unparseable URLs also return ``None``.
+ With ``reject_anchors``, a relative URL that gains an absolute or
+ drive-relative path raises ``ValueError`` instead of being skipped.
+ """
+ href = _URL_EDGE_C0_OR_SPACE_RE.sub("", href)
+ try:
+ parsed = urlsplit(href)
+ except ValueError:
+ return None
+ if parsed.scheme or parsed.netloc:
+ return None
+
+ # Preserve invalid UTF-8 bytes instead of aliasing distinct destinations to
+ # the replacement character. Scheme classification belongs to the raw URL.
+ path = unquote(parsed.path, errors="surrogateescape").replace("\\", "/")
+ if parsed.path.startswith("/"):
+ return None
+ normalized = posixpath.normpath(path)
+ if PureWindowsPath(normalized).anchor:
+ if reject_anchors:
+ raise ValueError("absolute or drive-relative path")
+ return None
+ if not allow_directory and path.endswith(("/", "/.", "/..")):
+ return None
+ return normalized
def _find_script_end(content: str, start: int) -> int | None:
@@ -97,7 +162,11 @@ class _AnchorHrefParser(HTMLParser):
self._non_navigation_tags: list[str] = []
def set_cdata_mode(self, elem: str, *, escapable: bool = False) -> None:
- super().set_cdata_mode(elem, escapable=escapable)
+ if _HTML_PARSER_SUPPORTS_ESCAPABLE_CDATA:
+ super().set_cdata_mode(elem, escapable=escapable)
+ else:
+ # ``escapable`` was added in CPython 3.12.12 and 3.13.6.
+ super().set_cdata_mode(elem)
if elem == "script":
self.interesting = _SCRIPT_END_SEARCH
@@ -153,14 +222,14 @@ def _walk_tokens(tokens: Iterable[Token]) -> Iterator[Token]:
yield from _walk_tokens(token.children)
-def markdown_link_targets(content: str) -> list[str]:
- """Return parser-normalized Markdown link destinations in document order."""
+def markdown_link_targets(content: str, *, include_images: bool = False) -> list[str]:
+ """Return parser-normalized Markdown destinations in document order."""
frontmatter = FRONTMATTER_PATTERN.match(content)
markdown_content = content
if frontmatter:
try:
frontmatter_data = yaml.safe_load(frontmatter.group(1))
- except yaml.YAMLError:
+ except (yaml.YAMLError, ValueError, RecursionError):
frontmatter_data = None
if isinstance(frontmatter_data, dict):
markdown_content = frontmatter.group(2)
@@ -171,6 +240,10 @@ def markdown_link_targets(content: str) -> list[str]:
href = token.attrGet("href")
if isinstance(href, str):
html_parts.append(f'<a href="{escape(href, quote=True)}"></a>')
+ elif include_images and token.type == "image":
+ source = token.attrGet("src")
+ if isinstance(source, str):
+ html_parts.append(f'<a href="{escape(source, quote=True)}"></a>')
elif token.type in {"html_inline", "html_block"}:
html_parts.append(token.content)
diff --git a/src/skillevaluator/validators/quality_score.py b/src/skillevaluator/validators/quality_score.py
index c43105f7..24395264 100644
--- a/src/skillevaluator/validators/quality_score.py
+++ b/src/skillevaluator/validators/quality_score.py
@@ -80,8 +80,12 @@ def _has_api_documentation(content: str) -> bool:
return any(re.search(pattern, content, re.IGNORECASE) for pattern in api_patterns)
-def _normalized_local_path(href: str) -> str | None:
- """Return a once-decoded, normalized local file path from a link destination."""
+def _normalized_quality_local_path(href: str) -> str | None:
+ """Normalize a local path using Quality Score's legacy classification order.
+
+ This intentionally remains separate from Hygiene's stricter path policy:
+ issue #108 requires this PR to leave existing Quality scores unchanged.
+ """
href = _URL_EDGE_C0_OR_SPACE_RE.sub("", href)
try:
parsed = urlsplit(href)
@@ -106,7 +110,7 @@ def _normalized_local_path(href: str) -> str | None:
def _has_nested_markdown_reference(content: str) -> bool:
"""Return whether a reference document links to another local Markdown document."""
for target in markdown_link_targets(content):
- path = _normalized_local_path(target)
+ path = _normalized_quality_local_path(target)
if path is None or not path.casefold().endswith(".md"):
continue
if path.casefold() == "../skill.md":
@@ -384,7 +388,7 @@ class QualityScoreValidator(ValidatorBase):
def _references_readme(content: str) -> bool:
"""Return whether SKILL.md contains a parsed Markdown link to README.md."""
for link_target in markdown_link_targets(content):
- path = _normalized_local_path(link_target)
+ path = _normalized_quality_local_path(link_target)
if path is not None and path.casefold() == "readme.md":
return True
return False
diff --git a/tests/test_cli_html_parser_compat.py b/tests/test_cli_html_parser_compat.py
new file mode 100644
index 00000000..3750dd1a
--- /dev/null
+++ b/tests/test_cli_html_parser_compat.py
@@ -0,0 +1,54 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Public CLI coverage for supported HTMLParser patch-version boundaries."""
+
+from pathlib import Path
+
+import pytest
+from click.testing import CliRunner
+
+from skillevaluator.cli import cli
+from skillevaluator.models.result import ValidationResult
+from skillevaluator.validators.code_risk import CodeRiskValidator
+from skillevaluator.validators.secrets import SecretsValidator
+
+
+def test_validate_code_integrity_handles_raw_html_on_supported_python_patches(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ skill = tmp_path / "html-parser-compatibility"
+ skill.mkdir()
+ (skill / "SKILL.md").write_text(
+ "---\n"
+ "name: html-parser-compatibility\n"
+ "description: Validate raw HTML link parsing on every supported Python patch.\n"
+ "---\n"
+ "\n"
+ "# HTML parser compatibility\n"
+ "\n"
+ "See the [supporting guide](guide.md).\n",
+ encoding="utf-8",
+ )
+ (skill / "guide.md").write_text(
+ "<script>const hidden = '<a href=\"missing-script.md\">';</script>\n"
+ '<textarea/><a href="missing-textarea.md">hidden</a></textarea>\n'
+ "See the [visible guide](visible.md).\n",
+ encoding="utf-8",
+ )
+ (skill / "visible.md").write_text("# Visible guide\n", encoding="utf-8")
+
+ # Exercise the public CLI and real Hygiene validator without invoking the
+ # independent scanner integrations.
+ monkeypatch.setattr(CodeRiskValidator, "validate", lambda _self, _path: ValidationResult())
+ monkeypatch.setattr(SecretsValidator, "validate", lambda _self, _path: ValidationResult())
+
+ result = CliRunner().invoke(
+ cli,
+ ["validate", str(skill), "--checks", "code-integrity", "--no-dedup", "-r", "cli"],
+ )
+
+ assert result.exit_code == 0, result.output
+ assert result.exception is None
+ assert "Traceback" not in result.output
diff --git a/tests/validators/test_hygiene.py b/tests/validators/test_hygiene.py
index ae1b7a07..72a3de1c 100644
--- a/tests/validators/test_hygiene.py
+++ b/tests/validators/test_hygiene.py
@@ -180,6 +180,99 @@ Check the [config file](./config/settings.yaml) for settings.
"dead link" in e.lower() or "docs/README.md" in e or "config/settings.yaml" in e for e in result.errors
)
+ @pytest.mark.parametrize(
+ ("content", "target"),
+ [
+ ("See [the guide][guide].\n\n[guide]: missing-reference.md\n", "missing-reference.md"),
+ ("See [guide][].\n\n[guide]: missing-collapsed.md\n", "missing-collapsed.md"),
+ ("See [guide].\n\n[guide]: missing-shortcut.md\n", "missing-shortcut.md"),
+ ('See <a href="missing-html.md">the guide</a>.\n', "missing-html.md"),
+ ("![diagram](missing-image.png)\n", "missing-image.png"),
+ ],
+ )
+ def test_commonmark_dead_links_are_detected(self, tmp_path: Path, content: str, target: str):
+ """Navigation and image targets discovered by CommonMark are checked."""
+ skill_dir = tmp_path / "commonmark-dead-links"
+ skill_dir.mkdir()
+ (skill_dir / "SKILL.md").write_text(content, encoding="utf-8")
+
+ result = HygieneValidator()._check_dead_links(skill_dir)
+
+ assert result.errors == [f"Dead link in SKILL.md: {target}"]
+
+ def test_commonmark_non_navigation_contexts_are_ignored(self, tmp_path: Path):
+ """Frontmatter, code, comments, and non-navigation HTML stay inert."""
+ skill_dir = tmp_path / "inert-links"
+ skill_dir.mkdir()
+ (skill_dir / "SKILL.md").write_text(
+ '---\ndescription: "[metadata](frontmatter.md)"\n---\n\n'
+ '`[inline](inline.md)`\n\n'
+ "```markdown\n[fenced](fenced.md)\n```\n\n"
+ "<!-- <a href='comment.md'>comment</a> -->\n"
+ "<template><a href='template.md'>template</a></template>\n",
+ encoding="utf-8",
+ )
+
+ result = HygieneValidator()._check_dead_links(skill_dir)
+
+ assert result.errors == []
+
+ def test_commonmark_local_targets_are_normalized_before_lookup(self, tmp_path: Path):
+ """Encoded spaces, queries, fragments, and directory links resolve locally."""
+ skill_dir = tmp_path / "normalized-links"
+ skill_dir.mkdir()
+ (skill_dir / "my file.md").write_text("# Existing\n", encoding="utf-8")
+ (skill_dir / "guide.md").write_text("# Existing\n", encoding="utf-8")
+ (skill_dir / "docs").mkdir()
+ (skill_dir / "SKILL.md").write_text(
+ "[space](<my file.md>)\n[query](guide.md?view=1)\n"
+ "[fragment](guide.md#overview)\n[directory](docs/)\n",
+ encoding="utf-8",
+ )
+
+ result = HygieneValidator()._check_dead_links(skill_dir)
+
+ assert result.errors == []
+
+ def test_missing_directory_target_is_reported(self, tmp_path: Path):
+ """Trailing slashes remain checkable for dead directory links."""
+ skill_dir = tmp_path / "missing-directory"
+ skill_dir.mkdir()
+ (skill_dir / "SKILL.md").write_text("[docs](missing-dir/)\n", encoding="utf-8")
+
+ result = HygieneValidator()._check_dead_links(skill_dir)
+
+ assert result.errors == ["Dead link in SKILL.md: missing-dir/"]
+
+ def test_non_local_and_root_absolute_targets_are_ignored(self, tmp_path: Path):
+ """External, anchor-only, and root-absolute destinations are not host paths."""
+ skill_dir = tmp_path / "non-local-links"
+ skill_dir.mkdir()
+ (skill_dir / "SKILL.md").write_text(
+ "[root](/config/settings.yaml)\n[protocol](//example.com/missing.md)\n"
+ "[web](https://example.com/missing.md)\n[email](mailto:[email protected])\n"
+ "[anchor](#overview)\n",
+ encoding="utf-8",
+ )
+
+ result = HygieneValidator()._check_dead_links(skill_dir)
+
+ assert result.errors == []
+
+ def test_repeated_dead_targets_are_reported_once(self, tmp_path: Path):
+ """Repeated links do not generate duplicate findings without line metadata."""
+ skill_dir = tmp_path / "duplicate-links"
+ skill_dir.mkdir()
+ (skill_dir / "SKILL.md").write_text(
+ "[first](missing.md), [second](./missing.md#part), and [third][same].\n\n"
+ "[same]: missing.md?view=1\n",
+ encoding="utf-8",
+ )
+
+ result = HygieneValidator()._check_dead_links(skill_dir)
+
+ assert result.errors == ["Dead link in SKILL.md: missing.md"]
+
def test_valid_relative_links_pass(self, tmp_path: Path):
"""Test that valid relative links pass."""
skill_dir = tmp_path / "valid-links-skill"
diff --git a/tests/validators/test_hygiene_link_boundaries.py b/tests/validators/test_hygiene_link_boundaries.py
new file mode 100644
index 00000000..93907358
--- /dev/null
+++ b/tests/validators/test_hygiene_link_boundaries.py
@@ -0,0 +1,217 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Caller-visible regressions for untrusted CommonMark destinations."""
+
+import re
+from functools import partial
+from pathlib import Path
+
+import pytest
+from rich.console import Console
+
+from skillevaluator.reporting import CLIReporter, MarkdownReporter
+from skillevaluator.reporting import cli as cli_reporting
+from skillevaluator.tier1.commands import run_validation
+from skillevaluator.validators import markdown as markdown_validator
+from skillevaluator.validators.base import ValidationResult
+from skillevaluator.validators.hygiene import HygieneValidator, _link_display
+from skillevaluator.validators.markdown import markdown_link_targets, normalized_local_path
+
+
+def _validate_document(tmp_path: Path, content: str):
+ """Scan a supporting document through the public Tier 1 orchestration."""
+ (tmp_path / "SKILL.md").write_text("---\nname: sample\ndescription: Example skill\n---\n", encoding="utf-8")
+ (tmp_path / "guide.md").write_text(content, encoding="utf-8")
+ results = run_validation(tmp_path, checks="code-integrity", content_type="skill")
+ return next(result for result in results if result.validator_name == HygieneValidator().name)
+
+
+def test_encoded_colon_stays_a_local_destination(tmp_path: Path) -> None:
+ result = _validate_document(tmp_path, "[guide][target]\n\n[target]: missing%3Aguide.md\n")
+ assert result.errors == ["Dead link in guide.md: missing%3Aguide.md"]
+ assert normalized_local_path("missing%3Aguide.md") == "missing:guide.md"
+
+
[email protected]("href", ["x/../C:/outside.md", "x/../C:outside.md", "C%3A/outside.md", "%5C%5Chost/share"])
+def test_normalized_windows_anchors_are_not_local(href: str) -> None:
+ assert normalized_local_path(href, allow_directory=True) is None
+
+
+ "href",
+ ["x/../C:/outside.md", "x/../C:outside.md", "C%3A/outside.md", "%5C%5Chost/share/outside.md", "%2Foutside.md"],
+)
+def test_unsafe_relative_anchors_produce_findings_without_lookup(
+ tmp_path: Path, href: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ assert normalized_local_path(href, allow_directory=True) is None
+ original_exists = Path.exists
+
+ def exists(path: Path) -> bool:
+ assert path.name not in {"outside.md", "C:outside.md"}
+ return original_exists(path)
+
+ monkeypatch.setattr(Path, "exists", exists)
+ result = _validate_document(tmp_path, f"[bad][target]\n\n[target]: {href}\n\n[other](missing.md)")
+ assert len(result.errors) == 2
+ assert result.errors[0] == f"Invalid local link in guide.md: {href} (absolute or drive-relative path)"
+ assert result.errors[1] == "Dead link in guide.md: missing.md"
+
+
+def test_repeated_unsafe_anchor_is_reported_once_per_document(tmp_path: Path) -> None:
+ content = (
+ '[first](C%3A/outside.md) <a href="C%3A/outside.md">second</a> [third][target]\n\n'
+ "[target]: C%3A/outside.md\n\n[other](missing.md)"
+ )
+ (tmp_path / "another.md").write_text(content, encoding="utf-8")
+ result = _validate_document(tmp_path, content)
+ assert sorted(result.errors) == sorted(
+ f"{message} in {name}: {target}"
+ for name in ("another.md", "guide.md")
+ for message, target in (
+ ("Invalid local link", "C%3A/outside.md (absolute or drive-relative path)"),
+ ("Dead link", "missing.md"),
+ )
+ )
+
+
+def test_invalid_utf8_does_not_alias_an_existing_replacement_character(tmp_path: Path) -> None:
+ (tmp_path / "\ufffd.md").write_text("Existing unrelated document", encoding="utf-8")
+ result = _validate_document(tmp_path, '<a href="%FF.md">one</a> <a href="%FE.md">two</a>')
+ assert result.errors == ["Dead link in guide.md: %FF.md", "Dead link in guide.md: %FE.md"]
+ assert normalized_local_path("%FF.md") != normalized_local_path("%FE.md")
+
+
[email protected]("value", ["9" * 5000, "[" * 1500 + "0" + "]" * 1500], ids=["huge-integer", "deep-sequence"])
+def test_frontmatter_limits_do_not_abort_validation(tmp_path: Path, value: str) -> None:
+ result = _validate_document(tmp_path, f"---\nvalue: {value}\n---\n[guide](missing.md)\n")
+ assert result.errors == ["Dead link in guide.md: missing.md"]
+
+
+def test_reference_lookup_failure_does_not_abort_other_links(tmp_path: Path) -> None:
+ target = "a" * 300 + ".md"
+ result = _validate_document(tmp_path, f"[long][target]\n\n[target]: {target}\n\n[other](missing.md)\n")
+ assert len(result.errors) == 2
+ assert result.errors[-1] == "Dead link in guide.md: missing.md"
+ assert len(result.errors[0]) < 220
+
+
+def test_lookup_oserror_is_contained_per_target(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ original_exists = Path.exists
+
+ def exists(path: Path) -> bool:
+ if path.name == "unreadable.md":
+ raise PermissionError("simulated inaccessible target")
+ return original_exists(path)
+
+ monkeypatch.setattr(Path, "exists", exists)
+ result = _validate_document(tmp_path, "[guide][target]\n\n[target]: unreadable.md\n\n[other](missing.md)")
+ assert result.errors == ["Dead link in guide.md: unreadable.md", "Dead link in guide.md: missing.md"]
+
+
+def test_lookup_programming_errors_are_not_silenced(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ original_exists = Path.exists
+
+ def exists(path: Path) -> bool:
+ if path.name == "broken.md":
+ raise RuntimeError("unexpected lookup bug")
+ return original_exists(path)
+
+ monkeypatch.setattr(Path, "exists", exists)
+ (tmp_path / "guide.md").write_text("[bad](broken.md)", encoding="utf-8")
+ with pytest.raises(RuntimeError, match="unexpected lookup bug"):
+ HygieneValidator()._check_dead_links(tmp_path)
+
+
[email protected]("prefix_length", range(155, 161))
[email protected]("control", ["\n", "\r", "\x1b", "\x00", "\u2028", "\U0001f600"])
[email protected]("reporter_type", [CLIReporter, MarkdownReporter])
+def test_diagnostic_truncation_cannot_restore_control_characters(
+ prefix_length: int, control: str, reporter_type: type, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ # Test injected line breaks separately from the CLI's intentional reflow.
+ monkeypatch.setattr(cli_reporting, "Console", partial(Console, width=1000, color_system="standard"))
+ display = _link_display("a" * prefix_length + control + "::error file=SKILL.md::forged")
+ assert len(display) <= 163
+ assert all(32 <= ord(character) < 127 for character in display)
+ result = ValidationResult()
+ result.add_error(f"Dead link: {display}")
+ output = reporter_type().render_all([result])
+ # The CLI deliberately emits SGR colors; do not mistake its styling for
+ # an injected escape. Other escape sequences must remain absent.
+ output = re.sub(r"\x1b\[[0-9;]*m", "", output)
+ assert not any(line.startswith("::") for line in output.splitlines())
+ assert "\x1b" not in output
+
+
[email protected]("reporter_type", [CLIReporter, MarkdownReporter])
+def test_link_diagnostics_cannot_inject_report_lines(tmp_path: Path, reporter_type: type) -> None:
+ result = _validate_document(tmp_path, '<a href="missing/&#10;::error file=SKILL.md,line=1::forged&#10;.md">bad</a>')
+ assert len(result.errors) == 1
+ assert "\\n::error" in result.errors[0]
+ assert len(result.errors[0].splitlines()) == 1
+ output = reporter_type().render_all([result])
+ assert not any(line.startswith("::") for line in output.splitlines())
+
+
[email protected]("kib", [8, 16, 32, 64])
+ "opener", ["<!--", "<?", "<![CDATA[", "<!a"], ids=["comment", "processing", "cdata", "declaration"]
+)
+def test_unclosed_html_scanning_is_linear(kib: int, opener: str, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Measure regex input work for the four guarded forms, not all Markdown."""
+ regex_input = 0
+ original_rule = markdown_validator.html_inline
+
+ def counted_rule(state, silent):
+ nonlocal regex_input
+ if state.src[state.pos] == "<":
+ regex_input += len(state.src) - state.pos
+ return original_rule(state, silent)
+
+ monkeypatch.setattr(markdown_validator, "html_inline", counted_rule)
+ content = '<a href="before.md">before</a>' + opener * (kib * 1024 // len(opener))
+ assert markdown_link_targets(content) == ["before.md"]
+ assert regex_input <= 2 * len(content)
+
+
+ "fragment",
+ [
+ '<!-- <a href="hidden.md">hidden</a> -->',
+ '<!-- <!-- <a href="hidden.md">hidden</a> -->',
+ "<!-->",
+ "<!--->",
+ '<?<a href="hidden.md">hidden</a>?>',
+ '<![CDATA[<a href="hidden.md">hidden</a>]]>',
+ '<!a <a href="hidden.md">',
+ ],
+)
+def test_html_guard_preserves_closed_fragments_and_later_links(fragment: str) -> None:
+ content = f"[first](first.md) {fragment} [last](last.md)\n\n" + "<!--" * 2
+ assert markdown_link_targets(content) == ["first.md", "last.md"]
+
+
+ "opener,closed",
+ [
+ ("<!--", '<!-- <a href="hidden.md">hidden</a> -->'),
+ ("<?", '<?<a href="hidden.md">hidden</a>?>'),
+ ("<![CDATA[", '<![CDATA[<a href="hidden.md">hidden</a>]]>'),
+ ("<!a", '<!a <a href="hidden.md">'),
+ ],
+)
+def test_html_guard_uses_each_inline_sources_offsets(opener: str, closed: str) -> None:
+ content = f"prefix {opener}\n\nnext [visible](visible.md) {closed}"
+ assert markdown_link_targets(content) == ["visible.md"]
+
+
[email protected]("reporter_type", [CLIReporter, MarkdownReporter])
+def test_link_diagnostics_bound_escaped_unicode(tmp_path: Path, reporter_type: type) -> None:
+ result = _validate_document(tmp_path, '<a href="missing/' + "\U0001f600" * 200 + '">bad</a>')
+ assert len(result.errors) == 1
+ assert result.errors[0].endswith("...")
+ assert len(result.errors[0]) < 220
+ assert "\\U0001f600" in result.errors[0]
+ assert len(reporter_type().render_all([result])) < 3000
diff --git a/tests/validators/test_markdown.py b/tests/validators/test_markdown.py
index 07e58c1d..a1788c01 100644
--- a/tests/validators/test_markdown.py
+++ b/tests/validators/test_markdown.py
@@ -6,7 +6,7 @@
import pytest
from skillevaluator.validators import markdown as markdown_validator
-from skillevaluator.validators.markdown import markdown_link_targets
+from skillevaluator.validators.markdown import markdown_link_targets, normalized_local_path
_CLOSED_NON_NAVIGATION_HTML_TAGS = (
"script",
@@ -40,6 +40,30 @@ def test_non_navigation_markdown_does_not_produce_links(content: str) -> None:
assert markdown_link_targets(content) == []
+ ("content", "target"),
+ [
+ ("![diagram](inline.png)", "inline.png"),
+ ("![diagram][asset]\n\n[asset]: reference.png", "reference.png"),
+ ],
+)
+def test_image_targets_are_opt_in(content: str, target: str) -> None:
+ assert markdown_link_targets(content) == []
+ assert markdown_link_targets(content, include_images=True) == [target]
+
+
+def test_opted_in_images_preserve_order_and_escaped_destinations() -> None:
+ content = "[first](first.md) ![diagram](a&b.png) [last](last.md)"
+
+ assert markdown_link_targets(content, include_images=True) == ["first.md", "a&b.png", "last.md"]
+
+
+def test_local_path_normalization_can_preserve_directory_targets() -> None:
+ assert normalized_local_path("docs/") is None
+ assert normalized_local_path("docs/", allow_directory=True) == "docs"
+ assert normalized_local_path("") == "."
+
+
@pytest.mark.parametrize(
("content", "target"),
[
@@ -70,6 +94,37 @@ def test_html_anchor_links_are_extracted(content: str, target: str) -> None:
assert markdown_link_targets(content) == [target]
+def test_cdata_mode_falls_back_to_legacy_stdlib_signature(monkeypatch: pytest.MonkeyPatch) -> None:
+ calls: list[str] = []
+
+ def legacy_set_cdata_mode(_parser, element: str) -> None:
+ calls.append(element)
+
+ monkeypatch.setattr(markdown_validator.HTMLParser, "set_cdata_mode", legacy_set_cdata_mode)
+ monkeypatch.setattr(markdown_validator, "_HTML_PARSER_SUPPORTS_ESCAPABLE_CDATA", False)
+ parser = markdown_validator._AnchorHrefParser()
+
+ parser.set_cdata_mode("script")
+ parser.set_cdata_mode("textarea", escapable=True)
+
+ assert calls == ["script", "textarea"]
+
+
+def test_cdata_mode_forwards_escapable_to_new_stdlib_signature(monkeypatch: pytest.MonkeyPatch) -> None:
+ calls: list[tuple[str, bool]] = []
+
+ def modern_set_cdata_mode(_parser, element: str, *, escapable: bool = False) -> None:
+ calls.append((element, escapable))
+
+ monkeypatch.setattr(markdown_validator.HTMLParser, "set_cdata_mode", modern_set_cdata_mode)
+ monkeypatch.setattr(markdown_validator, "_HTML_PARSER_SUPPORTS_ESCAPABLE_CDATA", True)
+ parser = markdown_validator._AnchorHrefParser()
+
+ parser.set_cdata_mode("textarea", escapable=True)
+
+ assert calls == [("textarea", True)]
+
+
def test_markdown_and_html_links_preserve_source_order() -> None:
content = '[first](first.md) <a href="second.md">second</a> [third](third.md)'
diff --git a/tests/validators/test_quality_score.py b/tests/validators/test_quality_score.py
index 89f62151..3abb6dc5 100644
--- a/tests/validators/test_quality_score.py
+++ b/tests/validators/test_quality_score.py
@@ -832,6 +832,60 @@ class TestQualityScoreDeterministicContracts:
assert all("Deeply nested references" not in message for message in messages)
+ @pytest.mark.parametrize(
+ "target",
+ [
+ "foo%3Abar.md",
+ "%66oo%3Abar.md",
+ "custom%2Bscheme%3Aguide.md",
+ ],
+ )
+ def test_percent_encoded_scheme_preserves_legacy_quality_score(self, tmp_path: Path, target: str):
+ skill_dir = _write_issue_skill(tmp_path)
+ references = skill_dir / "references"
+ references.mkdir()
+ (references / "mechanisms.md").write_text(f"See [more details]({target}).\n")
+
+ result = QualityScoreValidator(min_score=99).validate(skill_dir)
+
+ assert result.passed
+ assert result.metadata["quality_scores"]["overall_score"] == 100.0
+ assert all("Deeply nested references" not in finding.message for finding in result.findings)
+
+ @pytest.mark.parametrize(
+ "target",
+ [
+ "./foo%3Abar.md",
+ "dir/../foo%3Abar.md",
+ "foo%253Abar.md",
+ "x/../C:/outside.md",
+ ],
+ )
+ def test_once_decoded_local_paths_preserve_legacy_quality_score(self, tmp_path: Path, target: str):
+ skill_dir = _write_issue_skill(tmp_path)
+ references = skill_dir / "references"
+ references.mkdir()
+ (references / "mechanisms.md").write_text(f"See [more details]({target}).\n")
+
+ result = QualityScoreValidator(min_score=99).validate(skill_dir)
+
+ assert not result.passed
+ assert result.metadata["quality_scores"]["overall_score"] == 98.5
+ assert "Deeply nested references in mechanisms.md" in [finding.message for finding in result.findings]
+
+ def test_percent_encoded_scheme_does_not_normalize_into_readme_reference(self, tmp_path: Path):
+ skill_dir = _write_issue_skill(
+ tmp_path,
+ extra_body="\nSee [documentation](foo%3A/../README.md).\n",
+ )
+ (skill_dir / "README.md").write_text("# Human documentation\n")
+
+ result = QualityScoreValidator(min_score=99).validate(skill_dir)
+
+ assert result.passed
+ assert result.metadata["quality_scores"]["overall_score"] == 100.0
+ assert all("SKILL.md references README.md" not in finding.message for finding in result.findings)
+
@pytest.mark.parametrize(
"target",
[