Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(cli): report malformed policy files cleanly (#129)

Rio Yuefcd97ae7e0d
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d667034d..0fc1c927 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,6 +19,9 @@ All notable changes to SkillEvaluator are documented in this file.
### Fixed
+- Malformed, non-UTF-8, or unreadable bundled and custom policy files now
+ produce path-specific CLI errors instead of leaking raw parser or I/O errors
+ ([#128](https://github.com/NVIDIA/SkillEvaluator/issues/128)).
- `create-eval-dataset --refine` resolves Harbor trial case ids from persisted
`reward.json` `entry_id` metadata, using folder-name parsing only as an
unambiguous legacy fallback.
diff --git a/src/skillevaluator/validators/policy.py b/src/skillevaluator/validators/policy.py
index e3634359..89a09e9f 100644
--- a/src/skillevaluator/validators/policy.py
+++ b/src/skillevaluator/validators/policy.py
@@ -161,6 +161,19 @@ _KNOWN_TOP_LEVEL_KEYS = {"profile", "identity", "severity_overrides"}
_KNOWN_IDENTITY_KEYS = {"author_email_regex"}
+def _load_policy_yaml(path: Path) -> Any:
+ """Load policy YAML and normalize parser failures to the public contract."""
+ try:
+ with path.open(encoding="utf-8") as fh:
+ return yaml.safe_load(fh) or {}
+ except (yaml.YAMLError, UnicodeError, RecursionError) as exc:
+ raise ValueError(f"Invalid policy YAML in {path}: {exc}") from exc
+ except FileNotFoundError:
+ raise
+ except OSError as exc:
+ raise ValueError(f"Could not read policy file {path}: {exc}") from exc
+
+
def _warn_unknown_keys(data: dict[str, Any], known: set[str], context: str, source: str) -> None:
unknown = set(data) - known
if unknown:
@@ -227,8 +240,7 @@ def load_profile(name: str = DEFAULT_PROFILE_NAME) -> ValidationPolicy:
if not path.exists():
available = sorted(p.stem for p in PROFILES_DIR.glob("*.yaml")) if PROFILES_DIR.exists() else []
raise FileNotFoundError(f"Unknown profile {name!r}. Available bundled profiles: {available or '(none)'}")
- with path.open(encoding="utf-8") as fh:
- data = yaml.safe_load(fh) or {}
+ data = _load_policy_yaml(path)
return _policy_from_data(data, fallback_profile=name, source=path)
@@ -248,8 +260,7 @@ def load_policy_file(
raise FileNotFoundError(f"Custom policy file not found: {path}")
base = load_profile(base_profile)
- with path.open(encoding="utf-8") as fh:
- custom_data = yaml.safe_load(fh) or {}
+ custom_data = _load_policy_yaml(path)
custom = _policy_from_data(custom_data, fallback_profile=base.profile, source=path)
merged_overrides = dict(base.severity_overrides)
diff --git a/tests/test_commands.py b/tests/test_commands.py
index b9e28b50..a91abe09 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -93,6 +93,29 @@ def test_validate_fixture_no_llm() -> None:
assert "All validations passed" in result.output
+def test_validate_reports_malformed_policy_without_traceback(tmp_path: Path) -> None:
+ policy = tmp_path / "broken-policy.yaml"
+ policy.write_text("severity_overrides: [", encoding="utf-8")
+
+ result = CliRunner().invoke(
+ cli,
+ [
+ "validate",
+ str(FIXTURE),
+ "--no-llm",
+ "--no-dedup",
+ "--checks",
+ "schema",
+ "--policy",
+ str(policy),
+ ],
+ )
+
+ assert result.exit_code == 1
+ assert f"Invalid policy YAML in {policy}" in result.output
+ assert isinstance(result.exception, SystemExit)
+
+
def test_validate_prints_tier1_section_banner() -> None:
# The Tier 1 section is announced as it runs so it is visibly reported in
# CI logs (SkillEvaluator parity), not only inside the final combined report.
diff --git a/tests/test_nvidia_build_bridge.py b/tests/test_nvidia_build_bridge.py
index bcb108cc..5bf3d1ac 100644
--- a/tests/test_nvidia_build_bridge.py
+++ b/tests/test_nvidia_build_bridge.py
@@ -1646,6 +1646,7 @@ def test_slow_drip_request_body_obeys_an_absolute_deadline(
def test_partial_headers_time_out_without_exceeding_the_worker_bound(
bridge_services: _BridgeServices, monkeypatch: pytest.MonkeyPatch
) -> None:
+ header_timeout = bridge.REQUEST_HEADER_TIMEOUT_SECONDS
monkeypatch.setattr(bridge, "REQUEST_HEADER_TIMEOUT_SECONDS", 0.05)
clients = [
socket.create_connection(("127.0.0.1", bridge_services.port), timeout=2) for _ in range(bridge.MAX_WORKERS + 2)
@@ -1653,8 +1654,12 @@ def test_partial_headers_time_out_without_exceeding_the_worker_bound(
try:
for client in clients:
client.sendall(b"POST /v1/responses HTTP/1.1\r\nHost: 127.0.0.1")
- time.sleep(0.1)
assert bridge_services.bridge_server.active_workers <= bridge.MAX_WORKERS
+ for client in clients:
+ _read_raw_http_response(client)
+ assert bridge_services.bridge_server.wait_for_workers(timeout=2)
+ # The short timeout exercises partial headers, not the normal health check.
+ monkeypatch.setattr(bridge, "REQUEST_HEADER_TIMEOUT_SECONDS", header_timeout)
assert (
_request(
f"{bridge_services.url}/healthz",
diff --git a/tests/validators/test_policy.py b/tests/validators/test_policy.py
index ccfe7014..f6bffbae 100644
--- a/tests/validators/test_policy.py
+++ b/tests/validators/test_policy.py
@@ -8,8 +8,10 @@ from __future__ import annotations
from pathlib import Path
import pytest
+import yaml
from skillevaluator.models.result import Severity
+from skillevaluator.validators import policy as policy_module
from skillevaluator.validators.policy import (
DEFAULT_PROFILE_NAME,
ValidationPolicy,
@@ -46,6 +48,75 @@ def test_custom_policy_overlays_the_public_profile(tmp_path: Path) -> None:
assert policy.severity_for("SCHEMA", "author_missing", Severity.LOW) == Severity.HIGH
+def test_custom_policy_wraps_malformed_yaml(tmp_path: Path) -> None:
+ custom = tmp_path / "broken-policy.yaml"
+ custom.write_text("severity_overrides: [", encoding="utf-8")
+
+ with pytest.raises(ValueError) as exc_info:
+ load_policy_file(custom)
+
+ assert f"Invalid policy YAML in {custom}" in str(exc_info.value)
+ assert isinstance(exc_info.value.__cause__, yaml.YAMLError)
+
+
+def test_custom_policy_wraps_invalid_encoding(tmp_path: Path) -> None:
+ custom = tmp_path / "utf16-policy.yaml"
+ custom.write_bytes("profile: encoded\n".encode("utf-16"))
+
+ with pytest.raises(ValueError) as exc_info:
+ load_policy_file(custom)
+
+ assert f"Invalid policy YAML in {custom}" in str(exc_info.value)
+ assert isinstance(exc_info.value.__cause__, UnicodeDecodeError)
+
+
+def test_custom_policy_wraps_read_errors(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ custom = tmp_path / "unreadable-policy.yaml"
+ custom.write_text("profile: unreadable\n", encoding="utf-8")
+ original_open = Path.open
+
+ def deny_custom_policy(path: Path, *args: object, **kwargs: object):
+ if path == custom:
+ raise PermissionError("access denied")
+ return original_open(path, *args, **kwargs)
+
+ monkeypatch.setattr(Path, "open", deny_custom_policy)
+
+ with pytest.raises(ValueError) as exc_info:
+ load_policy_file(custom)
+
+ assert f"Could not read policy file {custom}" in str(exc_info.value)
+ assert isinstance(exc_info.value.__cause__, PermissionError)
+
+
+def test_policy_yaml_wraps_recursion_errors(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ policy = tmp_path / "deeply-nested-policy.yaml"
+ policy.write_text("severity_overrides: []\n", encoding="utf-8")
+
+ def exhaust_parser(_stream: object) -> object:
+ raise RecursionError("maximum recursion depth exceeded")
+
+ monkeypatch.setattr(policy_module.yaml, "safe_load", exhaust_parser)
+
+ with pytest.raises(ValueError) as exc_info:
+ policy_module._load_policy_yaml(policy)
+
+ assert f"Invalid policy YAML in {policy}" in str(exc_info.value)
+ assert isinstance(exc_info.value.__cause__, RecursionError)
+
+
+def test_bundled_profile_wraps_malformed_yaml(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ profile = tmp_path / "broken.yaml"
+ profile.write_text("identity: [", encoding="utf-8")
+ monkeypatch.setattr(policy_module, "PROFILES_DIR", tmp_path)
+
+ with pytest.raises(ValueError) as exc_info:
+ load_profile("broken")
+
+ assert f"Invalid policy YAML in {profile}" in str(exc_info.value)
+ assert isinstance(exc_info.value.__cause__, yaml.YAMLError)
+
+
def test_policy_validation_and_resolution() -> None:
assert resolve_policy().profile == "external"
assert (