Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(reporting): drop PII evidence digests that redaction made stale (#348)

Narendran Raghavan03e8c8bc01a3
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ec7f8f8a..2c01638e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,11 @@ All notable changes to SkillEvaluator are documented in this file.
## Unreleased
+### Changed
+
+- When CI redaction masks a credential inside a reported PII value, remove the
+ finding's digest as well.
+
## 0.8.0 - 2026-11-05
### Added
diff --git a/src/skillevaluator/reporting/finding_summary.py b/src/skillevaluator/reporting/finding_summary.py
index 2a523070..5aba85d7 100644
--- a/src/skillevaluator/reporting/finding_summary.py
+++ b/src/skillevaluator/reporting/finding_summary.py
@@ -217,7 +217,9 @@ def refresh_finding_identities(document: dict[str, Any]) -> None:
redacted value could be recovered by hashing guesses against them. Call
this after redacting: it replaces only the ID and reconciliation fields
the report already has, so a report written before finding IDs existed
- keeps its shape. Message counts do not depend on text and are kept.
+ keeps its shape. Message counts do not depend on text and are kept. A PII
+ ``evidence_sha256`` that no longer matches the redacted value is removed
+ for the same reason.
"""
rows = document.get("results")
if not isinstance(rows, list):
@@ -230,6 +232,7 @@ def refresh_finding_identities(document: dict[str, Any]) -> None:
for item in row["findings"]:
if not isinstance(item, dict):
continue
+ _drop_stale_evidence_digest(item)
finding = Finding(
category=item.get("category"),
severity=item.get("severity"),
@@ -256,6 +259,30 @@ def refresh_finding_identities(document: dict[str, Any]) -> None:
document["finding_groups"] = merged["groups"]
+def _drop_stale_evidence_digest(item: dict[str, Any]) -> None:
+ """Remove a PII finding's ``evidence_sha256`` unless it is the digest of the value the finding shows.
+
+ A finding carries the digest only while it reports its value in full.
+ Once redaction masks part of that value, the digest of the original would
+ let anyone holding the report confirm a guess for the masked part.
+ """
+ metadata = item.get("metadata")
+ if not isinstance(metadata, dict) or "evidence_sha256" not in metadata:
+ return
+ # Imported here so that importing the reporting package does not load every validator.
+ from skillevaluator.validators.pii_waivers import pii_evidence_sha256
+
+ value = metadata.get("matched_value")
+ check_name = item.get("check_name")
+ if (
+ isinstance(value, str)
+ and isinstance(check_name, str)
+ and metadata["evidence_sha256"] == pii_evidence_sha256(check_name, value)
+ ):
+ return
+ del metadata["evidence_sha256"]
+
+
def summarize_messages(results: list[ValidationResult]) -> dict[str, int]:
"""Count string messages that no structured finding represents.
diff --git a/tests/ci/test_secret_redaction.py b/tests/ci/test_secret_redaction.py
index 7e20868d..7cbedb81 100644
--- a/tests/ci/test_secret_redaction.py
+++ b/tests/ci/test_secret_redaction.py
@@ -10,6 +10,7 @@ import importlib.util
import io
import json
import subprocess
+import sys
import zipfile
from pathlib import Path
from urllib.parse import quote
@@ -358,6 +359,66 @@ def test_refreshing_ids_keeps_the_shape_of_reports_written_before_ids() -> None:
assert document == before
+def _pii_finding(value: str) -> Finding:
+ from skillevaluator.validators.pii_waivers import pii_evidence_sha256
+
+ return Finding(
+ category="PII",
+ severity=Severity.MEDIUM,
+ check_name="email",
+ message=f"Email address: {value}",
+ file_path="SKILL.md",
+ line_number=4,
+ line_content=f"Contact {value}",
+ metadata={"matched_value": value, "evidence_sha256": pii_evidence_sha256("email", value)},
+ )
+
+
+def test_redacting_a_reported_pii_value_drops_its_evidence_digest() -> None:
+ from skillevaluator.reporting.finding_summary import refresh_finding_identities
+ from skillevaluator.utils.secret_redaction import Redactor
+ from skillevaluator.validators.pii_waivers import pii_evidence_sha256
+
+ result = ValidationResult(validator_name="PII Scan")
+ result.add_structured_finding(_pii_finding(masked), is_error=False)
+ result.add_structured_finding(_pii_finding(kept), is_error=False)
+ document = Redactor({"sampletoken"}).object(json.loads(JSONReporter().render_all([result])))
+
+ refresh_finding_identities(document)
+
+ by_value = {f["metadata"]["matched_value"]: f["metadata"] for f in document["results"][0]["findings"]}
+ assert masked not in by_value
+ redacted = next(meta for value, meta in by_value.items() if value != kept)
+ assert "evidence_sha256" not in redacted
+ assert by_value[kept]["evidence_sha256"] == pii_evidence_sha256("email", kept)
+ rendered = json.dumps(document)
+ assert pii_evidence_sha256("email", masked) not in rendered
+ assert pii_evidence_sha256("email", kept) in rendered
+
+
+def test_a_digest_that_does_not_match_its_reported_value_is_dropped() -> None:
+ from skillevaluator.reporting.finding_summary import refresh_finding_identities
+
+ finding = _pii_finding("[email protected]").to_dict()
+ finding["metadata"]["evidence_sha256"] = "0" * 64
+ document = {"results": [{"validator": "PII Scan", "findings": [finding]}]}
+
+ refresh_finding_identities(document)
+
+ assert "evidence_sha256" not in document["results"][0]["findings"][0]["metadata"]
+
+
+def test_refreshing_finding_ids_does_not_load_the_validators() -> None:
+ probe = (
+ "import sys, skillevaluator.reporting.finding_summary; "
+ "print(sorted(m for m in sys.modules if m.startswith('skillevaluator.validators')))"
+ )
+ loaded = subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True, check=True).stdout
+
+ assert loaded.strip() == "[]"
+
+
def test_interrupted_scan_never_copies_raw_output(tmp_path, capsys):
workspace, reports = tmp_path / "workspace", tmp_path / "reports"
_skill(workspace)