Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(security): stop reporting User-Agent versions as IP addresses (#168)
Jeremy Levartovskyd0157411fe6c
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex a5ff6d62..60687241 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -6,6 +6,8 @@ All notable changes to SkillEvaluator are documented in this file.### Fixed+- Stop the PII scan reporting User-Agent product versions such as `Chrome/140.0.0.0`+ as public IP addresses. Chromium's reduced User-Agent gives every version this shape.- Separate Tier 2 collection limits from the 256-file per-skill limit. Freshsimilarity scans now allow 1,024 selected manifests and 128 million scalarcomparisons by default, covering 343 skills with 2,048-dimensional embeddings.diff --git a/src/skillevaluator/validators/security.py b/src/skillevaluator/validators/security.pyindex c18215de..a082a641 100644--- a/src/skillevaluator/validators/security.py+++ b/src/skillevaluator/validators/security.py@@ -2497,6 +2497,12 @@ class SecurityValidator(ValidatorBase):)_URL_AUTHORITY_PREFIX_PATTERN = re.compile(r"(?i)[a-z][a-z0-9+.-]*://[^/\s\"']*\Z")_IPV4_LITERAL_PATTERN = re.compile(r"(?<![\d.])(?:\d{1,3}\.){3}\d{1,3}(?![\d.])")+ # Chromium's reduced User-Agent reports every version as ``<major>.0.0.0``+ # after a product token, as in ``Chrome/140.0.0.0`` (or ``Chrome\/140.0.0.0``+ # in JSON with escaped slashes). A ``.0.0.0`` value is never a host address,+ # and a version is never followed by ``/``, so CIDR blocks stay reported.+ _USER_AGENT_PRODUCT_PREFIX_PATTERN = re.compile(r"[A-Za-z][A-Za-z0-9]*\\?/\Z")+ _REDUCED_USER_AGENT_VERSION_PATTERN = re.compile(r"\d{1,3}\.0\.0\.0")@staticmethoddef _is_near_zero_gps(line: str) -> bool:@@ -2521,6 +2527,13 @@ class SecurityValidator(ValidatorBase):):return True+ if (+ cls._REDUCED_USER_AGENT_VERSION_PATTERN.fullmatch(match.group())+ and cls._USER_AGENT_PRODUCT_PREFIX_PATTERN.search(prefix)+ and not line.startswith("/", match.end())+ ):+ return True+quoted_value: str | None = Nonequote_start = max(line.rfind('"', 0, match.start()), line.rfind("'", 0, match.start()))if quote_start >= 0:diff --git a/tests/validators/test_security.py b/tests/validators/test_security.pyindex 3cb6dbec..db575e3f 100644--- a/tests/validators/test_security.py+++ b/tests/validators/test_security.py@@ -5417,3 +5417,58 @@ class TestSpdxAndIpFalsePositiveHardening:result = SecurityValidator().validate_pii_only(skill_dir)assert any(finding.check_name == "ip_addresses" for finding in result.findings)++ @pytest.mark.parametrize(+ "line",+ [+ "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "+ "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36\n",+ "HeadlessChrome/151.0.0.0 -> Chrome/151.0.0.0\n",+ "Edge reports `Edg/140.0.0.0` and Opera reports OPR/122.0.0.0.\n",+ "page = browser.new_page(user_agent='Chrome/140.0.0.0', base_url=base)\n",+ '{"ua": "Mozilla\\/5.0 (X11; Linux x86_64) Chrome\\/140.0.0.0 Safari\\/537.36"}\n',+ ],+ )+ def test_user_agent_product_versions_are_not_pii(self, tmp_path: Path, line: str) -> None:+ skill_dir = tmp_path / "user-agent-skill"+ skill_dir.mkdir()+ (skill_dir / "SKILL.md").write_text(line, encoding="utf-8")++ result = SecurityValidator().validate_pii_only(skill_dir)++ assert not any(finding.check_name == "ip_addresses" for finding in result.findings)++ @pytest.mark.parametrize(+ "line",+ [+ "backups/8.8.8.8\n",+ "Backups/8.8.8.8\n",+ "whitelist: Office/52.14.1.9\n",+ "https://files.example.com/Chrome/8.8.8.8\n",+ "Proxy/8.8.8.8\n",+ "Allow 52.0.0.0 through the firewall.\n",+ "allowlist/52.0.0.0/8\n",+ "https://api.example.com/v1/ips/52.0.0.0/8\n",+ ],+ )+ def test_ip_outside_a_reduced_user_agent_version_remains_pii(self, tmp_path: Path, line: str) -> None:+ skill_dir = tmp_path / "slash-ip-skill"+ skill_dir.mkdir()+ (skill_dir / "SKILL.md").write_text(line, encoding="utf-8")++ result = SecurityValidator().validate_pii_only(skill_dir)++ assert any(finding.check_name == "ip_addresses" for finding in result.findings)++ def test_real_ip_on_a_user_agent_line_remains_pii(self, tmp_path: Path) -> None:+ skill_dir = tmp_path / "user-agent-and-ip-skill"+ skill_dir.mkdir()+ (skill_dir / "SKILL.md").write_text("Chrome/140.0.0.0 through proxy 8.8.8.8\n", encoding="utf-8")++ result = SecurityValidator().validate_pii_only(skill_dir)++ assert [+ finding.metadata.get("matched_value")+ for finding in result.findings+ if finding.check_name == "ip_addresses"+ ] == ["8.8.8.8"]