Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(hygiene): detect unpinned dependencies with environment markers (#107)

Rio Yuf2d8f5d2914e
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7da1a171..35b08cfa 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,10 @@ All notable changes to SkillEvaluator are documented in this file.
### Fixed
+- Unpinned-dependency warnings are no longer suppressed by comparison
+ operators inside PEP 508 environment markers; requirements such as
+ `pkg; python_version < "3.13"` are now correctly reported, while direct
+ references are treated as pinned independently of marker contents.
- SPDX headers keep the full license expression, so `MIT OR GPL-3.0` is
no longer truncated to MIT and allowed. Closing comment markers such as
`*/` and `-->` are not treated as part of the expression
diff --git a/src/skillevaluator/validators/hygiene.py b/src/skillevaluator/validators/hygiene.py
index 76cea316..6cb3653c 100644
--- a/src/skillevaluator/validators/hygiene.py
+++ b/src/skillevaluator/validators/hygiene.py
@@ -173,10 +173,23 @@ class HygieneValidator(ValidatorBase):
continue
pkg_name = match.group(1).lower()
+ # Pip strips inline comments introduced by whitespace before it
+ # parses a requirement. Remove them here as well so an "@" in a
+ # comment cannot be mistaken for a direct-reference separator;
+ # URL fragments remain intact because their "#" is not preceded
+ # by whitespace.
+ logical_line = re.split(r"\s+#", line, maxsplit=1)[0]
+
+ # Marker comparisons do not constrain the package version. Detect
+ # direct references before the marker so an "@" inside a marker
+ # value cannot hide an otherwise unpinned requirement.
+ requirement_part = logical_line.partition(";")[0]
+ _, direct_reference_separator, direct_reference_target = requirement_part.partition("@")
+ is_direct_reference = bool(direct_reference_separator and direct_reference_target.strip())
if pkg_name in banned_lower:
result.add_error(f"{req_file.name}:{line_num} - Banned package: {pkg_name}")
- elif not re.search(r"[=<>!]", line):
+ elif not is_direct_reference and not re.search(r"[=<>!]", requirement_part):
result.add_warning(f"{req_file.name}:{line_num} - Unpinned: {line}")
if not result.errors and not result.warnings:
diff --git a/tests/validators/test_hygiene.py b/tests/validators/test_hygiene.py
index 72a3de1c..6336ddba 100644
--- a/tests/validators/test_hygiene.py
+++ b/tests/validators/test_hygiene.py
@@ -360,6 +360,100 @@ pandas
all_messages = result.errors + result.warnings
assert any("unpinned" in m.lower() for m in all_messages)
+ @pytest.mark.parametrize(
+ "requirement",
+ [
+ 'requests; python_version < "3.13"',
+ 'requests; python_version != "3.12"',
+ 'requests;python_version<"3.13"',
+ 'requests ; python_version < "3.13"',
+ 'requests; sys_platform == "win32"',
+ 'requests; implementation_name == "cpython@corp"',
+ 'requests[security]; python_version >= "3.9"',
+ ],
+ )
+ def test_environment_marker_comparisons_do_not_hide_unpinned_requirements(self, tmp_path: Path, requirement: str):
+ """Marker operators must not count as package version constraints."""
+ requirements = tmp_path / "requirements.txt"
+ requirements.write_text(f"{requirement}\n", encoding="utf-8")
+
+ result = HygieneValidator()._check_requirements_file(requirements)
+
+ assert result.errors == []
+ assert result.warnings == [f"requirements.txt:1 - Unpinned: {requirement}"]
+
+ @pytest.mark.parametrize(
+ "requirement",
+ [
+ "requests # [email protected]",
+ "requests[security] # contact @ owner",
+ "requests # consider >=2 later",
+ ],
+ )
+ def test_inline_comments_do_not_hide_unpinned_requirements(self, tmp_path: Path, requirement: str):
+ """Pip-style inline comments must not affect constraint detection."""
+ requirements = tmp_path / "requirements.txt"
+ requirements.write_text(f"{requirement}\n", encoding="utf-8")
+
+ result = HygieneValidator()._check_requirements_file(requirements)
+
+ assert result.errors == []
+ assert result.warnings == [f"requirements.txt:1 - Unpinned: {requirement}"]
+
+ @pytest.mark.parametrize(
+ "requirement",
+ [
+ "requests @",
+ "requests @ # [email protected]",
+ 'requests @ ; python_version < "3.13"',
+ ],
+ )
+ def test_empty_direct_references_are_not_treated_as_pinned(self, tmp_path: Path, requirement: str):
+ """A direct-reference separator must be followed by a target."""
+ requirements = tmp_path / "requirements.txt"
+ requirements.write_text(f"{requirement}\n", encoding="utf-8")
+
+ result = HygieneValidator()._check_requirements_file(requirements)
+
+ assert result.errors == []
+ assert result.warnings == [f"requirements.txt:1 - Unpinned: {requirement}"]
+
+ @pytest.mark.parametrize(
+ "requirement",
+ [
+ 'requests>=2; python_version < "3.13"',
+ "requests>=2.0,<3.0",
+ "requests==2.31.0",
+ "requests~=2.31",
+ "requests!=2.30.0",
+ "requests>=2 # [email protected]",
+ "requests @ https://example.invalid/requests.whl",
+ "requests @ https://example.invalid/a;v=1/requests.whl",
+ "requests @ https://example.invalid/requests.whl#sha256=abc123",
+ "requests @ https://example.invalid/requests.whl # [email protected]",
+ 'requests @ https://example.invalid/requests.whl ; python_version < "3.13"',
+ ],
+ )
+ def test_marker_handling_preserves_constrained_and_direct_requirements(self, tmp_path: Path, requirement: str):
+ """Version constraints remain accepted, and direct references are treated as pinned."""
+ requirements = tmp_path / "requirements.txt"
+ requirements.write_text(f"{requirement}\n", encoding="utf-8")
+
+ result = HygieneValidator()._check_requirements_file(requirements)
+
+ assert result.errors == []
+ assert result.warnings == []
+
+ def test_banned_requirement_with_marker_remains_an_error(self, tmp_path: Path):
+ """Banned-package errors keep precedence over marker-aware warnings."""
+ requirements = tmp_path / "requirements.txt"
+ requirements.write_text('pycrypto; python_version < "3.13"\n', encoding="utf-8")
+
+ result = HygieneValidator()._check_requirements_file(requirements)
+
+ assert result.errors == ["requirements.txt:1 - Banned package: pycrypto"]
+ assert result.warnings == []
+
def test_detects_banned_packages(self, tmp_path: Path):
"""Test detection of banned/deprecated packages."""
skill_dir = tmp_path / "banned-deps-skill"