Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(reporting): drop PII evidence digests that redaction made stale (#348)
Narendran Raghavan03e8c8bc01a3
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ec7f8f8a..2c01638e 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -4,6 +4,11 @@ All notable changes to SkillEvaluator are documented in this file.## Unreleased+### Changed++- When CI redaction masks a credential inside a reported PII value, remove the+ finding's digest as well.+## 0.8.0 - 2026-11-05### Addeddiff --git a/src/skillevaluator/reporting/finding_summary.py b/src/skillevaluator/reporting/finding_summary.pyindex 2a523070..5aba85d7 100644--- a/src/skillevaluator/reporting/finding_summary.py+++ b/src/skillevaluator/reporting/finding_summary.py@@ -217,7 +217,9 @@ def refresh_finding_identities(document: dict[str, Any]) -> None:redacted value could be recovered by hashing guesses against them. Callthis after redacting: it replaces only the ID and reconciliation fieldsthe report already has, so a report written before finding IDs existed- keeps its shape. Message counts do not depend on text and are kept.+ keeps its shape. Message counts do not depend on text and are kept. A PII+ ``evidence_sha256`` that no longer matches the redacted value is removed+ for the same reason."""rows = document.get("results")if not isinstance(rows, list):@@ -230,6 +232,7 @@ def refresh_finding_identities(document: dict[str, Any]) -> None:for item in row["findings"]:if not isinstance(item, dict):continue+ _drop_stale_evidence_digest(item)finding = Finding(category=item.get("category"),severity=item.get("severity"),@@ -256,6 +259,30 @@ def refresh_finding_identities(document: dict[str, Any]) -> None:document["finding_groups"] = merged["groups"]+def _drop_stale_evidence_digest(item: dict[str, Any]) -> None:+ """Remove a PII finding's ``evidence_sha256`` unless it is the digest of the value the finding shows.++ A finding carries the digest only while it reports its value in full.+ Once redaction masks part of that value, the digest of the original would+ let anyone holding the report confirm a guess for the masked part.+ """+ metadata = item.get("metadata")+ if not isinstance(metadata, dict) or "evidence_sha256" not in metadata:+ return+ # Imported here so that importing the reporting package does not load every validator.+ from skillevaluator.validators.pii_waivers import pii_evidence_sha256++ value = metadata.get("matched_value")+ check_name = item.get("check_name")+ if (+ isinstance(value, str)+ and isinstance(check_name, str)+ and metadata["evidence_sha256"] == pii_evidence_sha256(check_name, value)+ ):+ return+ del metadata["evidence_sha256"]++def summarize_messages(results: list[ValidationResult]) -> dict[str, int]:"""Count string messages that no structured finding represents.diff --git a/tests/ci/test_secret_redaction.py b/tests/ci/test_secret_redaction.pyindex 7e20868d..7cbedb81 100644--- a/tests/ci/test_secret_redaction.py+++ b/tests/ci/test_secret_redaction.py@@ -10,6 +10,7 @@ import importlib.utilimport ioimport jsonimport subprocess+import sysimport zipfilefrom pathlib import Pathfrom urllib.parse import quote@@ -358,6 +359,66 @@ def test_refreshing_ids_keeps_the_shape_of_reports_written_before_ids() -> None:assert document == before+def _pii_finding(value: str) -> Finding:+ from skillevaluator.validators.pii_waivers import pii_evidence_sha256++ return Finding(+ category="PII",+ severity=Severity.MEDIUM,+ check_name="email",+ message=f"Email address: {value}",+ file_path="SKILL.md",+ line_number=4,+ line_content=f"Contact {value}",+ metadata={"matched_value": value, "evidence_sha256": pii_evidence_sha256("email", value)},+ )+++def test_redacting_a_reported_pii_value_drops_its_evidence_digest() -> None:+ from skillevaluator.reporting.finding_summary import refresh_finding_identities+ from skillevaluator.utils.secret_redaction import Redactor+ from skillevaluator.validators.pii_waivers import pii_evidence_sha256++ masked, kept = "[email protected]", "[email protected]"+ result = ValidationResult(validator_name="PII Scan")+ result.add_structured_finding(_pii_finding(masked), is_error=False)+ result.add_structured_finding(_pii_finding(kept), is_error=False)+ document = Redactor({"sampletoken"}).object(json.loads(JSONReporter().render_all([result])))++ refresh_finding_identities(document)++ by_value = {f["metadata"]["matched_value"]: f["metadata"] for f in document["results"][0]["findings"]}+ assert masked not in by_value+ redacted = next(meta for value, meta in by_value.items() if value != kept)+ assert "evidence_sha256" not in redacted+ assert by_value[kept]["evidence_sha256"] == pii_evidence_sha256("email", kept)+ rendered = json.dumps(document)+ assert pii_evidence_sha256("email", masked) not in rendered+ assert pii_evidence_sha256("email", kept) in rendered+++def test_a_digest_that_does_not_match_its_reported_value_is_dropped() -> None:+ from skillevaluator.reporting.finding_summary import refresh_finding_identities++ finding = _pii_finding("[email protected]").to_dict()+ finding["metadata"]["evidence_sha256"] = "0" * 64+ document = {"results": [{"validator": "PII Scan", "findings": [finding]}]}++ refresh_finding_identities(document)++ assert "evidence_sha256" not in document["results"][0]["findings"][0]["metadata"]+++def test_refreshing_finding_ids_does_not_load_the_validators() -> None:+ probe = (+ "import sys, skillevaluator.reporting.finding_summary; "+ "print(sorted(m for m in sys.modules if m.startswith('skillevaluator.validators')))"+ )+ loaded = subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True, check=True).stdout++ assert loaded.strip() == "[]"++def test_interrupted_scan_never_copies_raw_output(tmp_path, capsys):workspace, reports = tmp_path / "workspace", tmp_path / "reports"_skill(workspace)