Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(pii): recognize reserved placeholder email domains (#246)
Narendran Raghavandd35391dff25
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6d2a97f5..36e5fe4a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -39,6 +39,12 @@ All notable changes to SkillEvaluator are documented in this file.### Fixed+- PII email scans recognize all RFC 2606 reserved domains, including their+ subdomains, mixed-case forms, trailing dots, and prose ellipses. A local-part+ boundary prevents quadratic rescanning of long excluded addresses. Ordinary+ addresses on the same line and domains that merely resemble placeholders remain detectable+ ([#244](https://github.com/NVIDIA/SkillEvaluator/issues/244)).+- Tier 3 security grading preserves ordinary nested `/tmp` cleanup, includingchild operands with trailing slashes, unless the current or an earlierexecution action contains recognized symlink-creation syntax (`ln -s`,diff --git a/src/skillevaluator/config/pii_patterns.yaml b/src/skillevaluator/config/pii_patterns.yamlindex 06f85ec0..4058f33c 100644--- a/src/skillevaluator/config/pii_patterns.yaml+++ b/src/skillevaluator/config/pii_patterns.yaml@@ -52,9 +52,9 @@ home_paths:description: Personal Linux home directory pathsuggestion: "Use a generic path or an environment variable like $HOME"-# Email addresses (excluding placeholder/test domains and OData property names)+# Email addresses (excluding RFC 2606 domains, legacy test.com, and OData property names)emails:- - pattern: '[A-Za-z0-9._%+-]+@(?!([a-z0-9-]+\.)*example\.com|([a-z0-9-]+\.)*example\.org|([a-z0-9-]+\.)*test\.com|odata\.)[A-Za-z0-9.-]+\.[A-Za-z]{2,}'+ - pattern: '(?<![A-Za-z0-9._%+-])[A-Za-z0-9._%+-]+@(?!(?:[a-z0-9-]+\.)*(?:example\.(?:com|net|org)|test\.com|test|example|invalid|localhost)\.*(?![a-z0-9.-])|odata\.)[A-Za-z0-9.-]+\.[A-Za-z]{2,}'severity: highdescription: Non-placeholder email addresssuggestion: "Remove the address or use a placeholder like [email protected]"diff --git a/tests/validators/test_security.py b/tests/validators/test_security.pyindex 08b1edab..888a6350 100644--- a/tests/validators/test_security.py+++ b/tests/validators/test_security.py@@ -6,6 +6,8 @@import jsonimport osimport re+import subprocess+import sysfrom pathlib import Pathfrom unittest.mock import patch@@ -743,7 +745,7 @@ metadata:# Corporate Email Test-Contact: [email protected] for help.+Contact: [email protected] for help.""")validator = SecurityValidator()@@ -764,14 +766,14 @@ Contact: [email protected] for help.name: public-author-skilldescription: A public skill with contributor metadata and a body contact leakmetadata:- author: Example Contributor <[email protected]>+ author: Example Contributor <[email protected]>---# Public Author Skill## Instructions-Contact [email protected] for private support.+Contact [email protected] for private support.## Examples@@ -785,7 +787,7 @@ Run the documented workflow.assert not [finding for finding in schema_result.findings if finding.check_name == "author_format"]email_findings = [finding for finding in pii_result.findings if finding.check_name == "emails"]assert len(email_findings) == 1- assert email_findings[0].line_content == "Contact [email protected] for private support."+ assert email_findings[0].line_content == "Contact [email protected] for private support."@pytest.mark.parametrize("newline", ["\n", "\r\n"])def test_bom_prefixed_frontmatter_author_email_not_flagged_in_security_scan(@@ -801,14 +803,14 @@ Run the documented workflow."name: bom-public-author-skill","description: A public skill with contributor metadata and a body contact leak","metadata:",- " author: Example Contributor <[email protected]>",+ " author: Example Contributor <[email protected]>","---","","# Public Author Skill","","## Instructions","",- "Contact [email protected] for private support.",+ "Contact [email protected] for private support.","","## Examples","",@@ -824,7 +826,7 @@ Run the documented workflow.assert not [finding for finding in schema_result.findings if finding.check_name == "author_format"]email_findings = [finding for finding in pii_result.findings if finding.check_name == "emails"]assert len(email_findings) == 1- assert email_findings[0].line_content == "Contact [email protected] for private support."+ assert email_findings[0].line_content == "Contact [email protected] for private support."def test_unrelated_home_roots_not_flagged(self, tmp_path: Path):"""Unrelated /home roots stay unflagged without an organization allowlist."""@@ -976,7 +978,7 @@ metadata:# Service Account Test-Contact: [email protected]+Contact: [email protected]Workspace: /home/buildbot/releases/current/""")@@ -1056,15 +1058,81 @@ Call us at 555-123-4567 or +1-555-987-6543assert "emails" in piiassert "ssn" in pii- def test_email_pattern_exempts_only_placeholder_domains(self):- """Email regex domain exemptions are limited to neutral placeholders."""- email_pattern = SecurityValidator().pii_patterns["emails"][0]["pattern"]- exempt_domain_literals = {- f"{name}.{suffix}"- for name, suffix in re.findall(r"([a-z][a-z0-9-]*)\\\.(com|org|net)", email_pattern, re.IGNORECASE)- }+ @pytest.mark.parametrize(+ "domain",+ ["example.com", "example.net", "example.org", "test.com", "unit.test", "unit.example", "unit.invalid", "unit.localhost"],+ )+ @pytest.mark.parametrize(+ "domain_format",+ ["{domain}", "sub.{domain}", "{domain_upper}", "sub.{domain}.", "{domain}...", "sub.{domain_upper}..."],+ )+ def test_reserved_domain_emails_are_not_pii(self, tmp_path: Path, domain: str, domain_format: str):+ document = tmp_path / "guide.md"+ domain = domain_format.format(domain=domain, domain_upper=domain.upper())+ document.write_text(f"Fixture identity: fixture@{domain}\n", encoding="utf-8")++ result = SecurityValidator().validate_pii_only(document)++ assert not [finding for finding in result.findings if finding.check_name == "emails"]++ @pytest.mark.parametrize("domain", ["example.com", "example.net", "example.invalid"])+ @pytest.mark.parametrize("fixture_first", [True, False])+ @pytest.mark.parametrize("punctuation", ["", "..."])+ def test_placeholder_on_same_line_preserves_real_email(+ self, tmp_path: Path, domain: str, fixture_first: bool, punctuation: str+ ):+ document = tmp_path / "guide.md"+ addresses = [f"fixture@{domain}{punctuation}", "[email protected]"]+ if not fixture_first:+ addresses.reverse()+ document.write_text("Contact " + " or ".join(addresses) + "\n", encoding="utf-8")++ result = SecurityValidator().validate_pii_only(document)+ emails = [finding for finding in result.findings if finding.check_name == "emails"]++ assert [finding.metadata["matched_value"] for finding in emails] == ["[email protected]"]+ assert emails[0].severity == Severity.HIGH++ @pytest.mark.parametrize(+ "domain",+ [+ "company.com", "notexample.net", "example.net.company.com", "example.invalid.company.com",+ "example.com-extra.net", "test.company.com", "example.organic", "company.invalidity",+ "company.examplecorp", "localhost.company.com", "test.com.company.com",+ ],+ )+ def test_nonplaceholder_email_domains_remain_pii(self, tmp_path: Path, domain: str):+ document = tmp_path / "guide.md"+ address = f"contact@{domain}"+ document.write_text(f"Contact {address}\n", encoding="utf-8")++ result = SecurityValidator().validate_pii_only(document)+ emails = [finding for finding in result.findings if finding.check_name == "emails"]++ assert [finding.metadata["matched_value"] for finding in emails] == [address]+ assert emails[0].severity == Severity.HIGH++ def test_long_placeholder_email_finishes_and_preserves_real_email(self, tmp_path: Path):+ document = tmp_path / "long-guide.md"+ document.write_text("x" * 200_000 + "@example.invalid; [email protected]\n", encoding="utf-8")+ script = """+import json+import sys+from pathlib import Path+from skillevaluator.validators.security import SecurityValidator+result = SecurityValidator().validate_pii_only(Path(sys.argv[1]))+print(json.dumps([finding.metadata["matched_value"] for finding in result.findings if finding.check_name == "emails"]))+"""+ # Isolate the regex so a backtracking regression cannot hang pytest.+ completed = subprocess.run(+ [sys.executable, "-c", script, str(document)],+ capture_output=True,+ text=True,+ timeout=30,+ check=True,+ )- assert exempt_domain_literals == {"example.com", "example.org", "test.com"}+ assert json.loads(completed.stdout) == ["[email protected]"]def test_home_path_exceptions_are_limited_to_portable_paths(self):"""Home-path handling must not depend on organization-specific roots."""