Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix: decode Git roots as file names and keep PowerShell prefix findings (#312 review) (#320)

Narendran Raghavan8939d52e8082
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3cd0df23..da64cd8f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -211,8 +211,9 @@ All notable changes to SkillEvaluator are documented in this file.
### Fixed
- On Windows, repository root and source-link detection no longer break for
- repositories in non-ASCII folders: Git output is decoded as UTF-8 instead of
- the locale code page.
+ repositories in non-ASCII folders. Git's repository path is decoded as a file
+ name and its remote URL and branch as UTF-8, instead of with the locale code
+ page.
- CI quality thresholds apply to every parallel scan worker. Report overviews
and detailed reports surface failed quality gates ahead of advisory findings,
with links to the check that needs attention.
diff --git a/src/skillevaluator/plugin_component_risk.py b/src/skillevaluator/plugin_component_risk.py
index cd28c5ff..64b7106e 100644
--- a/src/skillevaluator/plugin_component_risk.py
+++ b/src/skillevaluator/plugin_component_risk.py
@@ -1641,10 +1641,11 @@ def _condition_scope(condition: str, shell_tools: tuple[str, ...] = ("Bash",)) -
return None
tool, content = match.group(1), match.group(2)
if tool in shell_tools:
- # A PowerShell rule is read as one (its launchers run any command); other shell-tool names as Bash.
- shell = _shell_tool(tool)
- rule = shell if content is None else f"{shell}({content})"
- return ("bash", ()) if bash_rule_risk(rule) is not None else ("narrow", ())
+ # A PowerShell rule is also read as one (its launchers, such as iex, run any command), on top of the
+ # Bash reading that catches prefixes every shell runs (sudo, env, xargs); other shell-tool names as Bash.
+ readings = {_shell_tool(tool), "Bash"}
+ broad = any(bash_rule_risk(name if content is None else f"{name}({content})") is not None for name in readings)
+ return ("bash", ()) if broad else ("narrow", ())
if tool in _SENSITIVE_TOOLS or tool.startswith("mcp__"):
label = "mcp__server__tool" if tool.startswith("mcp__") else tool
return "scoped", (_SENSITIVE_TOOL_LABELS.get(label, tool),)
diff --git a/src/skillevaluator/tier3/results_location.py b/src/skillevaluator/tier3/results_location.py
index 11a5aad5..c9516819 100644
--- a/src/skillevaluator/tier3/results_location.py
+++ b/src/skillevaluator/tier3/results_location.py
@@ -1094,14 +1094,14 @@ def git_root_for(path: Path) -> Path | None:
result = subprocess.run(
["git", "-C", str(path.expanduser().resolve()), "rev-parse", "--show-toplevel"],
capture_output=True,
- text=True,
check=False,
)
except (OSError, subprocess.SubprocessError):
return None
if result.returncode != 0:
return None
- root = result.stdout.strip()
+ # Raw file-name bytes: decode them the way Python encodes paths, not with the locale code page.
+ root = os.fsdecode(result.stdout).strip()
return Path(root).resolve() if root else None
diff --git a/src/skillevaluator/utils/helpers.py b/src/skillevaluator/utils/helpers.py
index fd47ae98..2b8e1055 100644
--- a/src/skillevaluator/utils/helpers.py
+++ b/src/skillevaluator/utils/helpers.py
@@ -125,14 +125,13 @@ def find_bundled_plugin_skills(plugin_root: Path) -> list[Path]:
]
-def _git_output(args: list[str], cwd: Path, *, errors: str = "replace") -> str:
+def _git_output(args: list[str], cwd: Path) -> str:
"""Run git in ``cwd`` and return its stripped stdout decoded as UTF-8.
- Git writes paths, refs and URLs as UTF-8 on every platform, so the locale
- code page (cp1252 on many Windows hosts) must not be used to decode them.
- ``errors`` is never ``"strict"``, so decoding cannot raise. The default
- ``"replace"`` suits display strings (URLs, refs) that are later written to
- UTF-8 reports, where surrogate escapes would fail to encode.
+ Remote URLs and ref names are UTF-8 text, so the locale code page (cp1252
+ on many Windows hosts) must not be used to decode them. Undecodable bytes
+ are replaced, so decoding cannot raise and the result can be written to
+ UTF-8 reports.
"""
return subprocess.check_output(
["git", *args],
@@ -140,7 +139,7 @@ def _git_output(args: list[str], cwd: Path, *, errors: str = "replace") -> str:
stderr=subprocess.DEVNULL,
text=True,
encoding="utf-8",
- errors=errors,
+ errors="replace",
).strip()
@@ -149,9 +148,13 @@ def resolve_git_root(local_path: Path) -> Path | None:
resolved = local_path.resolve()
working_dir = resolved if resolved.is_dir() else resolved.parent
try:
- # surrogateescape keeps non-UTF-8 path bytes losslessly (as POSIX
- # filesystem decoding does), so the result still names the real directory.
- root = _git_output(["rev-parse", "--show-toplevel"], working_dir, errors="surrogateescape")
+ # Git prints the root as raw file-name bytes; decode them the way Python
+ # encodes paths, not with the locale code page, so the result names the real directory.
+ root = os.fsdecode(
+ subprocess.check_output(
+ ["git", "rev-parse", "--show-toplevel"], cwd=str(working_dir), stderr=subprocess.DEVNULL
+ )
+ ).strip()
except (subprocess.CalledProcessError, FileNotFoundError, OSError):
return None
return Path(root).resolve() if root else None
diff --git a/src/skillevaluator/utils/path_security.py b/src/skillevaluator/utils/path_security.py
index ad731cd0..0feb8df1 100644
--- a/src/skillevaluator/utils/path_security.py
+++ b/src/skillevaluator/utils/path_security.py
@@ -85,12 +85,13 @@ def find_git_repo_root(path: Path) -> Path | None:
result = subprocess.run(
["git", "-C", str(path), "rev-parse", "--show-toplevel"],
capture_output=True,
- text=True,
timeout=5,
check=False,
)
- if result.returncode == 0 and result.stdout.strip():
- return Path(result.stdout.strip()).resolve()
+ # Raw file-name bytes: decode them the way Python encodes paths, not with the locale code page.
+ root = os.fsdecode(result.stdout).strip()
+ if result.returncode == 0 and root:
+ return Path(root).resolve()
except Exception:
pass
diff --git a/tests/utils/test_helpers.py b/tests/utils/test_helpers.py
index 425b2d6a..8a36180c 100644
--- a/tests/utils/test_helpers.py
+++ b/tests/utils/test_helpers.py
@@ -3,6 +3,7 @@
"""Tests for skillevaluator.utils module."""
+import os
import subprocess
import sys
from pathlib import Path
@@ -10,12 +11,14 @@ from unittest.mock import patch
import pytest
+from skillevaluator.tier3.results_location import git_root_for
from skillevaluator.utils import find_skills_in_directory, get_skill_name_from_path
from skillevaluator.utils.helpers import (
_ssh_to_https,
resolve_git_remote_url,
resolve_git_root,
)
+from skillevaluator.utils.path_security import find_git_repo_root
class TestFindSkillsInDirectory:
@@ -192,6 +195,16 @@ def _cp1252_locale_check_output(*args, **kwargs):
return _REAL_CHECK_OUTPUT(*args, **kwargs)
+_REAL_RUN = subprocess.run
+
+
+def _cp1252_locale_run(*args, **kwargs):
+ """Run the real command, decoding text output as a Windows cp1252 locale would."""
+ if kwargs.get("text") and kwargs.get("encoding") is None:
+ kwargs["encoding"] = "cp1252"
+ return _REAL_RUN(*args, **kwargs)
+
+
def _fake_git_output(outputs: dict[str, bytes]):
"""Return a check_output stand-in that emits raw git bytes decoded like a cp1252 locale."""
@@ -233,7 +246,7 @@ class TestResolveGitRoot:
@pytest.mark.skipif(sys.platform == "win32", reason="non-UTF-8 path bytes are a POSIX filesystem case")
def test_non_utf8_root_bytes_round_trip(self, tmp_path: Path, monkeypatch) -> None:
"""Undecodable path bytes are kept losslessly instead of raising or being replaced."""
- raw_root = str(tmp_path.resolve()).encode("utf-8") + b"/repo-\xff"
+ raw_root = os.fsencode(str(tmp_path.resolve())) + b"/repo-\xff"
monkeypatch.setattr(
subprocess, "check_output", _fake_git_output({"rev-parse --show-toplevel": raw_root + b"\n"})
)
@@ -241,7 +254,19 @@ class TestResolveGitRoot:
root = resolve_git_root(tmp_path)
assert root is not None
- assert str(root).encode("utf-8", "surrogateescape") == raw_root
+ assert os.fsencode(str(root)) == raw_root
+
+
[email protected]("lookup", [resolve_git_root, find_git_repo_root, git_root_for], ids=lambda f: f.__name__)
+def test_every_git_root_lookup_names_a_non_ascii_root(tmp_path: Path, monkeypatch, lookup) -> None:
+ """Each repository-root lookup decodes git's path bytes as file names, not with the locale code page."""
+ repo_root = tmp_path / "café-skills"
+ _init_git_repo(repo_root, "[email protected]:example/project.git")
+ (repo_root / "skills").mkdir()
+ monkeypatch.setattr(subprocess, "check_output", _cp1252_locale_check_output)
+ monkeypatch.setattr(subprocess, "run", _cp1252_locale_run)
+
+ assert lookup(repo_root / "skills") == repo_root.resolve()
class TestResolveGitRemoteUrl:
diff --git a/tests/validators/test_plugin_component_risk.py b/tests/validators/test_plugin_component_risk.py
index a8fa8e10..cd522aa2 100644
--- a/tests/validators/test_plugin_component_risk.py
+++ b/tests/validators/test_plugin_component_risk.py
@@ -368,6 +368,8 @@ def _conditional_approve_hook(tmp_path: Path, matcher: str, condition: str) -> V
"PowerShell(pwsh *)",
"PowerShell(cmd.exe *)",
"PowerShell(start-process *)",
+ "PowerShell(sudo *)",
+ "PowerShell(env *)",
],
)
def test_auto_approve_with_a_powershell_launcher_condition_is_high(tmp_path: Path, condition: str) -> None: