Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(reporting): write reports for non-UTF-8 file names and keep SARIF URIs valid (#536)
Narendran Raghavan550bb735233a
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex bab2333f..4a5524f4 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -264,6 +264,18 @@ All notable changes to SkillEvaluator are documented in this file.- SARIF `artifactLocation.uri` values percent-encode a `%` in a file name as`%25`. A file such as `a%20b.md` no longer shares the URI of `a b.md`, and`100%.md` no longer produces an invalid URI.+- Reports no longer fail on Linux when a skill file name is not valid UTF-8+ (#525). Finding IDs, HTML issue keys, the written HTML, JSON, Markdown and+ SARIF reports, the CI workflow scripts that re-render retained reports, and+ generated `evals/evals.json` datasets accept the surrogate-escaped name+ Python gives such a file. The SARIF `artifactLocation.uri` percent-encodes+ the name's original bytes (`caf%E9.md`).+- SARIF `artifactLocation.uri` values stay valid for more paths. A+ repository-relative path whose first segment contains a colon, such as+ `notes:v2.md`, gets a `./` prefix so it no longer reads as a URI scheme. A+ path outside the workspace is written as an absolute `file:` URI instead of+ a bare file system path, and a symlink in the workspace that points outside+ it keeps the link's repository-relative path.- Accuracy and Behavior judge prompts clarify that explicit final-answer formatsapply to the entire final response, independently of correct values in tooloutput or references. Scoring and thresholds are unchanged; an Accuracy formatdiff --git a/ci/native_reports.py b/ci/native_reports.pyindex 7ee402dd..9da059e1 100644--- a/ci/native_reports.py+++ b/ci/native_reports.py@@ -90,9 +90,9 @@ def _constrain_native_html(document: str, trusted_code: frozenset[str]) -> str:code = {*parser.scripts, *parser.handlers}if parser.external_script or not code <= trusted_code:raise ValueError("Unexpected executable markup in the native report")- hashes = sorted(- {"'sha256-" + base64.b64encode(hashlib.sha256(code.encode()).digest()).decode() + "'" for code in code}- )+ # Hash the bytes the report is written with (report_views._write escapes lone surrogates).+ digests = (hashlib.sha256(item.encode("utf-8", "backslashreplace")).digest() for item in code)+ hashes = sorted({"'sha256-" + base64.b64encode(digest).decode() + "'" for digest in digests})policy = ("default-src 'none'; script-src 'unsafe-hashes' "+ " ".join(hashes)diff --git a/ci/report_views.py b/ci/report_views.pyindex 81f5314a..62d9519c 100644--- a/ci/report_views.py+++ b/ci/report_views.py@@ -300,7 +300,9 @@ def _write(path: Path, content: str) -> None:raise ValueError("Refusing to replace a symlink report view.")temporary = Nonetry:- with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent, delete=False) as handle:+ with tempfile.NamedTemporaryFile(+ mode="w", encoding="utf-8", errors="backslashreplace", dir=path.parent, delete=False+ ) as handle:temporary = Path(handle.name)handle.write(content)temporary.replace(path)diff --git a/ci/run_tiers.py b/ci/run_tiers.pyindex b45c3ba5..267da880 100644--- a/ci/run_tiers.py+++ b/ci/run_tiers.py@@ -379,10 +379,13 @@ def _retain(context={"report_index_url": "../../index.html", "canonical_json_url": f"{name}.json"},),encoding="utf-8",+ errors="backslashreplace",)from skillevaluator.reporting.markdown import MarkdownReporter- (destination / f"{name}.md").write_text(MarkdownReporter().render_all(results), encoding="utf-8")+ (destination / f"{name}.md").write_text(+ MarkdownReporter().render_all(results), encoding="utf-8", errors="backslashreplace"+ )metrics = {"findings": sum(len(result.findings) for result in results)}if tier == 3:payload = report.get("tier3") or {}diff --git a/ci/secret_redaction.py b/ci/secret_redaction.pyindex 3b345762..415380e8 100644--- a/ci/secret_redaction.py+++ b/ci/secret_redaction.py@@ -156,7 +156,8 @@ def sanitized_outputs(raw: Path, tokens: set[str], *, target_path: str | None =if name.endswith(".md")else SARIFReporter())- sanitized[name] = reporter.render_all(results).encode("utf-8")+ # Undecodable file names come back as lone surrogates; escape them.+ sanitized[name] = reporter.render_all(results).encode("utf-8", "backslashreplace")continuetext = content.decode("utf-8")if name.endswith(".json"):diff --git a/src/skillevaluator/models/result.py b/src/skillevaluator/models/result.pyindex e3ee89a0..6012770a 100644--- a/src/skillevaluator/models/result.py+++ b/src/skillevaluator/models/result.py@@ -203,7 +203,7 @@ class Finding:self._normalize_identity_text(self.suggestion),))- digest = hashlib.sha256(identity.encode("utf-8"), usedforsecurity=False).hexdigest()[:20]+ digest = hashlib.sha256(identity.encode("utf-8", "surrogatepass"), usedforsecurity=False).hexdigest()[:20]return f"SEF-{digest}"def location_occurrence_id(self, file_path: object, line_number: object, line_content: object = None) -> str:@@ -222,7 +222,7 @@ class Finding:self._normalize_identity_text(line_content),))- digest = hashlib.sha256(identity.encode("utf-8"), usedforsecurity=False).hexdigest()[:20]+ digest = hashlib.sha256(identity.encode("utf-8", "surrogatepass"), usedforsecurity=False).hexdigest()[:20]return f"SEO-{digest}"@propertydiff --git a/src/skillevaluator/reporting/base.py b/src/skillevaluator/reporting/base.pyindex d6609da2..ac687f47 100644--- a/src/skillevaluator/reporting/base.py+++ b/src/skillevaluator/reporting/base.py@@ -2708,7 +2708,9 @@ class ReporterBase(ABC):Raises:UnsafeReportPathError: If the destination cannot be written safely."""- payload = self.render_all(results).encode("utf-8")+ # A file name that is not valid UTF-8 reaches messages as lone+ # surrogates; write those as escapes instead of failing the report.+ payload = self.render_all(results).encode("utf-8", "backslashreplace")_write_report_atomically(output_path, payload)def get_file_extension(self) -> str:diff --git a/src/skillevaluator/reporting/html.py b/src/skillevaluator/reporting/html.pyindex 05a10b15..75108667 100644--- a/src/skillevaluator/reporting/html.py+++ b/src/skillevaluator/reporting/html.py@@ -2417,7 +2417,7 @@ class HTMLReporter(ReporterBase):16 hex chars (~64 bits) is well below collision risk for the fewdozen unique issue groups a single report ever shows."""- payload = f"{category}::{group_key}".encode()+ payload = f"{category}::{group_key}".encode("utf-8", "surrogatepass")return hashlib.sha1(payload, usedforsecurity=False).hexdigest()[:16]@staticmethoddiff --git a/src/skillevaluator/reporting/sarif_reporter.py b/src/skillevaluator/reporting/sarif_reporter.pyindex fbd3a2e4..d0d8c79b 100644--- a/src/skillevaluator/reporting/sarif_reporter.py+++ b/src/skillevaluator/reporting/sarif_reporter.py@@ -11,6 +11,7 @@ validators without structured findings are omitted.from __future__ import annotationsimport json+import osimport refrom datetime import UTC, datetimefrom pathlib import Path@@ -100,29 +101,102 @@ def _resolve_artifact_path(file_path: str, scan_root: Path | None) -> Path:return path+def _quote_uri_path(text: str, safe: str = "/:@") -> str:+ """Percent-encode a POSIX-style path for use in a URI.++ A file name that is not valid UTF-8 arrives with surrogate escapes; its+ original bytes are encoded (``caf\\udce9.md`` becomes ``caf%E9.md``).+ """+ try:+ octets = text.encode("utf-8", "surrogateescape")+ except UnicodeEncodeError:+ octets = text.encode("utf-8", "surrogatepass")+ return quote(octets, safe=safe)+++def _relative_reference(normalized: str) -> str:+ """Return a relative URI reference whose first segment cannot read as a scheme."""+ uri = _quote_uri_path(normalized)+ # RFC 3986 section 4.2: a colon in the first segment would make+ # ``notes:v2.md`` parse as scheme ``notes``; a dot segment keeps it a path.+ if ":" in uri.split("/", 1)[0]:+ return f"./{uri}"+ return uri+++def _file_uri(path: Path) -> str:+ """Return an absolute ``file:`` URI for a path outside the workspace."""+ posix = path.as_posix()+ if posix.startswith("//"): # UNC: //server/share/...+ host, _, rest = posix[2:].partition("/")+ return f"file://{_quote_uri_path(host, safe='')}/{_quote_uri_path(rest)}"+ if posix.startswith("/"):+ return f"file://{_quote_uri_path(posix)}"+ return f"file:///{_quote_uri_path(posix)}" # drive letter: C:/...+++def _unresolved_artifact_path(file_path: str, scan_root: Path | None) -> Path | None:+ """Return the absolute path a finding names, without following its symlinks.++ ``None`` when the path has a ``..`` segment: after a symlinked directory+ that names a different file than the lexically normalized path would.+ """+ path = Path(file_path.replace("\\", "/"))+ if ".." in path.parts:+ return None+ if not path.is_absolute():+ if scan_root is None:+ return None+ path = scan_root.resolve() / path+ return Path(os.path.normpath(path))+++def _workspace_relative_path(+ path: Path,+ file_path: str,+ workspace_root: Path,+ scan_root: Path | None,+) -> Path | None:+ """Return the artifact path relative to the workspace root, or ``None``.++ The resolved path is tried first. A symlink inside the workspace whose+ target is outside it keeps the path of the link, which is what the+ finding is about.+ """+ root = workspace_root.resolve()+ candidates = [path.resolve()]+ unresolved = _unresolved_artifact_path(file_path, scan_root)+ if unresolved is not None:+ candidates.append(unresolved)+ for candidate in candidates:+ try:+ return candidate.relative_to(root)+ except ValueError:+ continue+ return None++def _normalize_artifact_uri(file_path: str,workspace_root: Path | None,scan_root: Path | None = None,) -> str:- """Return a repository-relative, URI-encoded artifact path for SARIF."""+ """Return a URI-encoded SARIF artifact location.++ Paths inside the workspace become repository-relative references; other+ absolute paths become ``file:`` URIs.+ """path = _resolve_artifact_path(file_path, scan_root)- if workspace_root is not None:- try:- # Windows paths can be rooted (``\\workspace\\...``) without a- # drive, in which case ``is_absolute()`` is false until resolved.- if path.is_absolute() or path.root:- relative = path.resolve().relative_to(workspace_root.resolve())- normalized = relative.as_posix()- else:- normalized = path.as_posix()- except ValueError:- normalized = path.as_posix()- elif path.is_absolute():- normalized = path.as_posix()- else:- normalized = path.as_posix()- return quote(normalized, safe="/:@")+ # Windows paths can be rooted (``\\workspace\\...``) without a drive, in+ # which case ``is_absolute()`` is false until resolved.+ if workspace_root is not None and (path.is_absolute() or path.root):+ relative = _workspace_relative_path(path, file_path, workspace_root, scan_root)+ if relative is not None:+ return _relative_reference(relative.as_posix())+ return _file_uri(path.resolve())+ if path.is_absolute():+ return _file_uri(path)+ return _relative_reference(path.as_posix())def _physical_location(diff --git a/src/skillevaluator/tier3/generate_dataset.py b/src/skillevaluator/tier3/generate_dataset.pyindex d2b9deb2..0b302b23 100644--- a/src/skillevaluator/tier3/generate_dataset.py+++ b/src/skillevaluator/tier3/generate_dataset.py@@ -748,7 +748,8 @@ def generate_one_case(cases=cases,)evals_dir.mkdir(parents=True, exist_ok=True)- with output_path.open("x", encoding="utf-8") as output:+ # backslashreplace writes an undecodable script name as a JSON \u escape.+ with output_path.open("x", encoding="utf-8", errors="backslashreplace") as output:json.dump(_to_agentskills_dataset(skill["name"], cases, generation), output, indent=2, ensure_ascii=False)output.write("\n")return output_path@@ -1084,7 +1085,7 @@ def _run_agent_collect_trajectories(),skill_path.name,)- with output_path.open("w", encoding="utf-8") as f:+ with output_path.open("w", encoding="utf-8", errors="backslashreplace") as f:json.dump(_to_agentskills_dataset(str(skill_name), cases), f, indent=2, ensure_ascii=False)print(f" Wrote initial {len(cases)} case(s) to {output_path}")@@ -1482,7 +1483,7 @@ def _write_dataset(output_path: Path, dataset: dict[str, Any]) -> None:output_path.parent.mkdir(parents=True, exist_ok=True)existing_mode = stat.S_IMODE(output_path.stat().st_mode) if output_path.exists() else Nonetemporary_path = output_path.with_name(f".{output_path.name}.{secrets.token_hex(16)}.tmp")- with temporary_path.open("x", encoding="utf-8") as temporary:+ with temporary_path.open("x", encoding="utf-8", errors="backslashreplace") as temporary:json.dump(dataset, temporary, indent=2, ensure_ascii=False)temporary.flush()os.fsync(temporary.fileno())diff --git a/tests/ci/test_secret_redaction.py b/tests/ci/test_secret_redaction.pyindex 2820a71e..5144765f 100644--- a/tests/ci/test_secret_redaction.py+++ b/tests/ci/test_secret_redaction.py@@ -925,3 +925,35 @@ def test_published_json_redacts_known_credentials_in_repository_relative_target_assert document["target_path"] == "skills/[REDACTED]/demo"assert token.encode() not in outputs["skillevaluator-tier1.json"]assert (raw / "skillevaluator-tier1.json").read_text(encoding="utf-8") == raw_document+++ sys.platform == "win32", reason="only POSIX file names carry undecodable bytes as surrogate escapes"+)+def test_reports_rendered_from_redacted_json_accept_an_undecodable_file_name(tmp_path):+ # On Linux, os.listdir returns the name b"caf\xe9.md" as "caf\udce9.md".+ name = "references/caf\udce9.md"+ result = ValidationResult(validator_name="Schema", validator_description="Check skill manifest")+ result.add_finding(+ Finding(+ category="SCHEMA",+ severity=Severity.LOW,+ check_name="unexpected_file",+ message=f"Unexpected '{name}' in skill root",+ file_path=name,+ )+ )+ raw = tmp_path / "raw"+ raw.mkdir()+ JSONReporter(include_timestamp=False, explicit_skill_name="demo").save([result], raw / "skillevaluator-tier1.json")+ for view in ("skillevaluator-tier1.html", "skillevaluator-tier1.md", "skillevaluator-tier1.sarif.json"):+ (raw / view).write_text("placeholder\n", encoding="utf-8")++ outputs = runner.sanitized_outputs(raw, set())++ for view in ("skillevaluator-tier1.html", "skillevaluator-tier1.md"):+ assert "Unexpected" in outputs[view].decode("utf-8")+ sarif = json.loads(outputs["skillevaluator-tier1.sarif.json"])+ assert sarif["runs"][0]["results"][0]["locations"][0]["physicalLocation"]["artifactLocation"]["uri"] == (+ "references/caf%E9.md"+ )diff --git a/tests/reporting/test_reporters.py b/tests/reporting/test_reporters.pyindex 66c76524..cdbae98b 100644--- a/tests/reporting/test_reporters.py+++ b/tests/reporting/test_reporters.py@@ -5,9 +5,11 @@import copyimport json+import osimport reimport tempfile-from pathlib import Path+from pathlib import Path, PureWindowsPath+from urllib.parse import unquote, urlsplitimport pytest@@ -27,6 +29,7 @@ from skillevaluator.reporting import (MarkdownReporter,SARIFReporter,)+from skillevaluator.reporting.sarif_reporter import _file_uri, _normalize_artifact_urifrom skillevaluator.reporting.summary_copy import SKILL_LIFT_DEFINITIONfrom skillevaluator.tier1.commands import emit_reports@@ -1660,6 +1663,97 @@ class TestSARIFReporter:uris = [item["locations"][0]["physicalLocation"]["artifactLocation"]["uri"] for item in results]assert uris == ["notes/a%20b.md", "notes/a%2520b.md", "notes/100%25.md"]+ @staticmethod+ def _artifact_uris(+ file_paths: list[str],+ *,+ workspace_root: Path | None,+ scan_root: Path | None = None,+ ) -> list[str]:+ result = ValidationResult(validator_name="QUALITY", validator_description="quality")+ for file_path in file_paths:+ result.add_finding(+ Finding(+ category="QUALITY",+ severity=Severity.MEDIUM,+ check_name="spacing",+ message="issue",+ file_path=file_path,+ line_number=1,+ )+ )+ reporter = SARIFReporter(include_timestamp=False, workspace_root=workspace_root, scan_root=scan_root)+ results = json.loads(reporter.render_all([result]))["runs"][0]["results"]+ return [item["locations"][0]["physicalLocation"]["artifactLocation"]["uri"] for item in results]++ @pytest.mark.skipif(os.name == "nt", reason="Windows file names cannot contain ':'")+ def test_physical_location_uri_keeps_a_colon_in_the_first_segment_a_path(self, tmp_path: Path) -> None:+ names = ["notes:v2.md", "skills/a:b.md"]+ uris = self._artifact_uris([str(tmp_path / name) for name in names], workspace_root=tmp_path)++ assert uris == ["./notes:v2.md", "skills/a:b.md"]+ assert [urlsplit(uri).scheme for uri in uris] == ["", ""]++ @pytest.mark.skipif(os.name == "nt", reason="only POSIX file names carry undecodable bytes as surrogate escapes")+ def test_physical_location_uri_encodes_the_bytes_of_an_undecodable_file_name(self, tmp_path: Path) -> None:+ uris = self._artifact_uris([str(tmp_path / "notes" / "caf\udce9.md")], workspace_root=tmp_path)++ assert uris == ["notes/caf%E9.md"]++ def test_physical_location_uri_outside_the_workspace_is_a_file_uri(self, tmp_path: Path) -> None:+ repo = tmp_path / "repo"+ repo.mkdir()+ outside = tmp_path / "outside" / "x y%.md"++ (uri,) = self._artifact_uris([str(outside)], workspace_root=repo)++ parts = urlsplit(uri)+ assert parts.scheme == "file"+ assert unquote(parts.path).endswith("/outside/x y%.md")++ def test_file_uri_for_windows_drive_and_unc_paths(self) -> None:+ assert _file_uri(PureWindowsPath(r"C:\outside\x y.md")) == "file:///C:/outside/x%20y.md"+ assert _file_uri(PureWindowsPath(r"\\server\share\x.md")) == "file://server/share/x.md"+ assert _file_uri(PureWindowsPath(r"\\server@SSL@443\DavWWWRoot\x.md")) == (+ "file://server%40SSL%40443/DavWWWRoot/x.md"+ )++ def test_physical_location_uri_encodes_a_lone_surrogate_that_is_not_an_escape(self) -> None:+ assert _normalize_artifact_uri("notes/\ud800.md", None) == "notes/%ED%A0%80.md"++ @pytest.mark.skipif(os.name == "nt", reason="Windows collapses '..' before it follows a symlink")+ def test_physical_location_uri_follows_dot_dot_through_a_symlinked_directory(self, tmp_path: Path) -> None:+ skill = tmp_path / "repo" / "skills" / "demo"+ skill.mkdir(parents=True)+ (skill / "x.md").write_text("decoy\n", encoding="utf-8")+ outside = tmp_path / "outside" / "deep"+ (outside / "dir").mkdir(parents=True)+ (outside / "x.md").write_text("real\n", encoding="utf-8")+ try:+ (skill / "dl").symlink_to(outside / "dir", target_is_directory=True)+ except OSError:+ pytest.skip("symlinks are not available")++ (uri,) = self._artifact_uris(["dl/../x.md"], workspace_root=tmp_path / "repo", scan_root=skill)++ parts = urlsplit(uri)+ assert parts.scheme == "file"+ assert unquote(parts.path).endswith("/outside/deep/x.md")++ def test_physical_location_uri_keeps_a_symlink_that_points_outside_the_workspace(self, tmp_path: Path) -> None:+ skill = tmp_path / "repo" / "skills" / "demo"+ skill.mkdir(parents=True)+ target = tmp_path / "outside.md"+ target.write_text("notes\n", encoding="utf-8")+ try:+ (skill / "notes.md").symlink_to(target)+ except OSError:+ pytest.skip("symlinks are not available")++ uris = self._artifact_uris(["notes.md"], workspace_root=tmp_path / "repo", scan_root=skill)++ assert uris == ["skills/demo/notes.md"]+def test_merge_catalog_sarif_combines_into_single_run(self) -> None:child = SARIFReporter(include_timestamp=False)documents = []@@ -3020,3 +3114,40 @@ def test_cli_harbor_job_labels_remain_neutral_and_literal(colored):assert "Job 3 [bad]" in console.export_text()if colored:assert "\x1b[1mJob 3 [bad]\x1b[0m" in output++[email protected](os.name == "nt", reason="only POSIX file names carry undecodable bytes as surrogate escapes")[email protected]("report_format", ["json", "markdown", "html", "sarif"])+def test_reports_are_written_for_a_file_name_that_is_not_utf8(tmp_path: Path, report_format: str) -> None:+ # On Linux, os.listdir returns the name b"caf\xe9.md" as "caf\udce9.md".+ skill = tmp_path / "skill"+ skill.mkdir()+ (skill / "SKILL.md").write_text("---\nname: skill\ndescription: Demo.\n---\n", encoding="utf-8")+ name = "caf\udce9.md"+ result = ValidationResult(validator_name="SCHEMA", validator_description="schema")+ result.add_finding(+ Finding(+ category="SCHEMA",+ severity=Severity.LOW,+ check_name="unexpected_file",+ message=f"Unexpected '{name}' in skill root",+ file_path=str(skill / name),+ )+ )+ output_dir = tmp_path / "reports"++ emit_reports(+ [result],+ report_formats=(report_format,),+ output_dir=output_dir,+ target_path=str(skill),+ announce_paths=False,+ sarif_scan_root=skill,+ sarif_repository_root=tmp_path,+ )++ (report,) = [path for path in output_dir.iterdir() if path.name != "BENCHMARK.md"]+ text = report.read_text(encoding="utf-8")+ assert "Unexpected" in text+ if report_format == "sarif":+ assert "skill/caf%E9.md" in textdiff --git a/tests/tier3/test_dataset_baseline_fairness.py b/tests/tier3/test_dataset_baseline_fairness.pyindex 99dc174d..d729bb74 100644--- a/tests/tier3/test_dataset_baseline_fairness.py+++ b/tests/tier3/test_dataset_baseline_fairness.py@@ -1748,3 +1748,13 @@ def test_adversarial_text_is_scanned_in_bounded_time(text):_, elapsed = _timed(lambda: _baseline_fairness_warnings(entry, skill_scripts=["review.py"]))assert elapsed < _TIME_LIMIT_S+++def test_dataset_write_keeps_an_undecodable_script_name(tmp_path):+ # On Linux, os.listdir returns the name b"r\xe9port.py" as "r\udce9port.py".+ dataset = {"cases": [{"id": "one", "expected_script": "r\udce9port.py"}]}+ output = tmp_path / "evals" / "evals.json"++ generate_dataset._write_dataset(output, dataset)++ assert json.loads(output.read_text(encoding="utf-8")) == dataset