Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
import os
from pathlib import Path
import pytest
from skillevaluator.utils import secure_fs
from skillevaluator.utils.secure_fs import (
SecurePathError,
SecureRoot,
discover_secure_files,
)
pytestmark = pytest.mark.skipif(os.name != "posix", reason="descriptor lifecycle tests are POSIX-specific")
def test_secure_root_closes_final_descriptor_when_post_open_fstat_fails(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
root = tmp_path / "unique-secure-root"
root.mkdir()
real_open = os.open
real_fstat = os.fstat
real_close = os.close
root_fd: int | None = None
fstat_calls = 0
closed: list[int] = []
def tracked_open(path, flags, mode=0o777, *, dir_fd=None):
nonlocal root_fd
if dir_fd is None:
descriptor = real_open(path, flags, mode)
else:
descriptor = real_open(path, flags, mode, dir_fd=dir_fd)
if Path(path) == root:
root_fd = descriptor
return descriptor
def failing_fstat(descriptor: int):
nonlocal fstat_calls
if descriptor == root_fd:
fstat_calls += 1
if fstat_calls == 2:
raise OSError("simulated post-open root fstat failure")
return real_fstat(descriptor)
def tracked_close(descriptor: int) -> None:
closed.append(descriptor)
real_close(descriptor)
monkeypatch.setattr(secure_fs.os, "open", tracked_open)
monkeypatch.setattr(secure_fs.os, "fstat", failing_fstat)
monkeypatch.setattr(secure_fs.os, "close", tracked_close)
with pytest.raises(OSError, match="post-open"), SecureRoot(root):
pass
assert root_fd is not None
assert root_fd in closed
def test_absolute_directory_open_closes_child_when_fstat_fails(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
root = tmp_path / "unique-fstat-root"
root.mkdir()
real_open = os.open
real_fstat = os.fstat
real_close = os.close
child_fd: int | None = None
closed: list[int] = []
def tracked_open(path, flags, mode=0o777, *, dir_fd=None):
nonlocal child_fd
if dir_fd is None:
descriptor = real_open(path, flags, mode)
else:
descriptor = real_open(path, flags, mode, dir_fd=dir_fd)
if Path(path) == root:
child_fd = descriptor
return descriptor
def failing_fstat(descriptor: int):
if descriptor == child_fd:
raise OSError("simulated child fstat failure")
return real_fstat(descriptor)
def tracked_close(descriptor: int) -> None:
closed.append(descriptor)
real_close(descriptor)
monkeypatch.setattr(secure_fs.os, "open", tracked_open)
monkeypatch.setattr(secure_fs.os, "fstat", failing_fstat)
monkeypatch.setattr(secure_fs.os, "close", tracked_close)
with pytest.raises(OSError, match="child fstat"):
secure_fs._open_absolute_directory_posix(root)
assert child_fd is not None
assert child_fd in closed
def test_secure_root_allows_symlinked_ancestor_when_declared_root_is_regular(tmp_path: Path) -> None:
real_parent = tmp_path / "real-parent"
root = real_parent / "root"
root.mkdir(parents=True)
(root / "guide.md").write_text("safe")
linked_parent = tmp_path / "linked-parent"
linked_parent.symlink_to(real_parent, target_is_directory=True)
with SecureRoot(linked_parent / "root") as secure_root:
assert secure_root.read_bytes(Path("guide.md"), 1024)[0] == b"safe"
def test_secure_root_rejects_symlinked_declared_root(tmp_path: Path) -> None:
real_root = tmp_path / "real-root"
real_root.mkdir()
linked_root = tmp_path / "linked-root"
linked_root.symlink_to(real_root, target_is_directory=True)
with pytest.raises(SecurePathError, match=r"root|symlink|reparse"), SecureRoot(linked_root):
pass
def test_secure_root_rejects_double_enter_without_replacing_descriptor(tmp_path: Path) -> None:
root = tmp_path / "root"
root.mkdir()
secure_root = SecureRoot(root)
secure_root.__enter__()
original_descriptor = secure_root._root_fd
try:
with pytest.raises(SecurePathError, match=r"already|active|entered"):
secure_root.__enter__()
assert secure_root._root_fd == original_descriptor
finally:
secure_root.__exit__(None, None, None)
def test_secure_root_read_requires_active_context(tmp_path: Path) -> None:
root = tmp_path / "root"
root.mkdir()
(root / "guide.md").write_text("safe")
with pytest.raises(SecurePathError, match=r"context|active|entered"):
SecureRoot(root).read_bytes(Path("guide.md"), 1024)
def test_component_open_closes_child_when_fstat_fails(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
root = tmp_path / "root"
references = root / "references"
references.mkdir(parents=True)
(references / "guide.md").write_text("safe")
real_open = os.open
real_fstat = os.fstat
real_close = os.close
child_fd: int | None = None
closed: list[int] = []
with SecureRoot(root) as secure_root:
def tracked_open(path, flags, mode=0o777, *, dir_fd=None):
nonlocal child_fd
if dir_fd is None:
descriptor = real_open(path, flags, mode)
else:
descriptor = real_open(path, flags, mode, dir_fd=dir_fd)
if str(path) == "references":
child_fd = descriptor
return descriptor
def failing_fstat(descriptor: int):
if descriptor == child_fd:
raise OSError("simulated component fstat failure")
return real_fstat(descriptor)
def tracked_close(descriptor: int) -> None:
closed.append(descriptor)
real_close(descriptor)
monkeypatch.setattr(secure_fs.os, "open", tracked_open)
monkeypatch.setattr(secure_fs.os, "fstat", failing_fstat)
monkeypatch.setattr(secure_fs.os, "close", tracked_close)
with pytest.raises(OSError, match="component fstat"):
secure_root.read_bytes(Path("references/guide.md"), 1024)
assert child_fd is not None
assert child_fd in closed
def test_same_inode_mutation_during_read_fails_closed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
root = tmp_path / "root"
root.mkdir()
target = root / "guide.md"
target.write_text("SAFE-CONTENT")
files = discover_secure_files(root, selected=lambda path: path.suffix == ".md", max_paths=10)
real_read = os.read
mutated = False
def mutating_read(descriptor: int, count: int) -> bytes:
nonlocal mutated
if not mutated:
target.write_text("EVIL-CONTENT")
metadata = files[0].metadata
os.utime(target, ns=(metadata.st_atime_ns, metadata.st_mtime_ns + 2_000_000_000))
mutated = True
return real_read(descriptor, count)
monkeypatch.setattr(secure_fs.os, "read", mutating_read)
with SecureRoot(root) as secure_root, pytest.raises(SecurePathError, match=r"changed|identity|mutation"):
secure_root.read_bytes(files[0].relative_path, 1024, expected=files[0].metadata)
assert mutated
assert target.stat().st_ino == files[0].metadata.st_ino
assert target.stat().st_size == files[0].metadata.st_size
def test_directory_swap_to_external_link_is_never_traversed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
root = tmp_path / "root"
child = root / "child"
child.mkdir(parents=True)
(child / "safe.md").write_text("safe")
outside = tmp_path / "outside"
outside.mkdir()
(outside / "secret.md").write_text("SECRET_CANARY")
moved = root / "moved-child"
real_open = os.open
swapped = False
selected_paths: list[str] = []
def swapping_open(path, flags, mode=0o777, *, dir_fd=None):
nonlocal swapped
if path == "child" and dir_fd is not None and flags & os.O_DIRECTORY and not swapped:
child.rename(moved)
child.symlink_to(outside, target_is_directory=True)
swapped = True
if dir_fd is None:
return real_open(path, flags, mode)
return real_open(path, flags, mode, dir_fd=dir_fd)
def selected(relative: Path) -> bool:
selected_paths.append(relative.as_posix())
return relative.suffix == ".md"
monkeypatch.setattr(secure_fs.os, "open", swapping_open)
with pytest.raises(SecurePathError, match=r"directory|link|changed|unsafe"):
discover_secure_files(root, selected=selected, max_paths=20)
assert swapped
assert "child/secret.md" not in selected_paths
def test_path_budget_stops_scandir_at_limit_plus_one(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
root = tmp_path / "root"
root.mkdir()
for index in range(100):
(root / f"irrelevant-{index:03}.bin").write_bytes(b"x")
real_scandir = os.scandir
yielded = 0
class TrackingScandir:
def __init__(self, path) -> None:
self._iterator = real_scandir(path)
def __enter__(self):
self._iterator.__enter__()
return self
def __exit__(self, *args):
return self._iterator.__exit__(*args)
def __iter__(self):
return self
def __next__(self):
nonlocal yielded
entry = next(self._iterator)
yielded += 1
return entry
monkeypatch.setattr(secure_fs.os, "scandir", TrackingScandir)
with pytest.raises(SecurePathError, match=r"path.*limit"):
discover_secure_files(root, selected=lambda _relative: False, max_paths=2)
assert yielded == 3
def test_selected_link_never_queries_target_type(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
root = tmp_path / "root"
root.mkdir()
(root / "guide.md").symlink_to("missing-target")
real_scandir = os.scandir
target_queries: list[str] = []
class EntryProxy:
def __init__(self, entry) -> None:
self._entry = entry
self.name = entry.name
def stat(self, *, follow_symlinks=True):
return self._entry.stat(follow_symlinks=follow_symlinks)
def is_dir(self, *, follow_symlinks=True):
if follow_symlinks:
target_queries.append(self.name)
raise AssertionError("selected link target metadata must not be queried")
return self._entry.is_dir(follow_symlinks=False)
class ScandirProxy:
def __init__(self, path) -> None:
self._iterator = real_scandir(path)
def __enter__(self):
self._iterator.__enter__()
return self
def __exit__(self, *args):
return self._iterator.__exit__(*args)
def __iter__(self):
return self
def __next__(self):
return EntryProxy(next(self._iterator))
monkeypatch.setattr(secure_fs.os, "scandir", ScandirProxy)
with pytest.raises(SecurePathError, match=r"symlink|reparse|unsafe"):
discover_secure_files(root, selected=lambda relative: relative.suffix == ".md", max_paths=10)
assert target_queries == []
def test_selected_directory_is_rejected_as_non_regular(tmp_path: Path) -> None:
root = tmp_path / "root"
(root / "guide.md").mkdir(parents=True)
with pytest.raises(SecurePathError, match=r"regular file|not a regular"):
discover_secure_files(root, selected=lambda relative: relative.suffix == ".md", max_paths=10)
def test_path_budget_exact_boundary_and_limit_metadata(tmp_path: Path) -> None:
root = tmp_path / "root"
child = root / "child"
child.mkdir(parents=True)
(child / "guide.md").write_text("safe")
files = discover_secure_files(root, selected=lambda relative: relative.suffix == ".md", max_paths=2)
assert [file.rel_path for file in files] == ["child/guide.md"]
(root / "extra.bin").write_bytes(b"x")
with pytest.raises(SecurePathError) as caught:
discover_secure_files(root, selected=lambda relative: relative.suffix == ".md", max_paths=2)
assert caught.value.metadata == {"actual": 3, "limit": 2}
def test_deep_tree_opens_each_directory_once_and_bounds_live_descriptors(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
root = tmp_path / "root"
root.mkdir()
current = root
directory_depth = 12
for _index in range(directory_depth):
current /= "d"
current.mkdir()
(current / "leaf.md").write_text("safe")
real_open = os.open
real_close = os.close
directory_open_count = 0
active_directory_descriptors: set[int] = set()
max_active_directory_descriptors = 0
def tracked_open(path, flags, mode=0o777, *, dir_fd=None):
nonlocal directory_open_count, max_active_directory_descriptors
if dir_fd is None:
descriptor = real_open(path, flags, mode)
else:
descriptor = real_open(path, flags, mode, dir_fd=dir_fd)
if flags & os.O_DIRECTORY:
directory_open_count += 1
active_directory_descriptors.add(descriptor)
max_active_directory_descriptors = max(
max_active_directory_descriptors,
len(active_directory_descriptors),
)
return descriptor
def tracked_close(descriptor: int) -> None:
active_directory_descriptors.discard(descriptor)
real_close(descriptor)
monkeypatch.setattr(secure_fs.os, "open", tracked_open)
monkeypatch.setattr(secure_fs.os, "close", tracked_close)
files = discover_secure_files(root, selected=lambda relative: relative.suffix == ".md", max_paths=30)
assert [file.relative_path.name for file in files] == ["leaf.md"]
assert directory_open_count == directory_depth + 1
assert max_active_directory_descriptors == directory_depth + 1
assert active_directory_descriptors == set()
def test_default_directory_depth_limit_fails_with_metadata(tmp_path: Path) -> None:
root = tmp_path / "root"
root.mkdir()
current = root
for _index in range(secure_fs.MAX_SECURE_DIRECTORY_DEPTH + 1):
current /= "d"
current.mkdir()
with pytest.raises(SecurePathError) as caught:
discover_secure_files(root, selected=lambda _relative: False, max_paths=100)
assert caught.value.code == "directory_depth_limit"
assert caught.value.metadata == {
"actual": secure_fs.MAX_SECURE_DIRECTORY_DEPTH + 1,
"limit": secure_fs.MAX_SECURE_DIRECTORY_DEPTH,
}
def test_child_entry_replacement_after_subtree_is_rejected(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
root = tmp_path / "root"
child = root / "child"
child.mkdir(parents=True)
(child / "safe.md").write_text("safe")
moved_child = root / "moved-child"
real_stat = os.stat
child_entry_stats = 0
swapped = False
def swapping_stat(path, *, dir_fd=None, follow_symlinks=True):
nonlocal child_entry_stats, swapped
if path == "child" and dir_fd is not None and not follow_symlinks:
child_entry_stats += 1
if child_entry_stats == 2:
child.rename(moved_child)
child.mkdir()
swapped = True
if dir_fd is None:
return real_stat(path, follow_symlinks=follow_symlinks)
return real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks)
monkeypatch.setattr(secure_fs.os, "stat", swapping_stat)
with pytest.raises(SecurePathError, match=r"directory|changed|identity|unsafe"):
discover_secure_files(root, selected=lambda relative: relative.suffix == ".md", max_paths=10)
assert swapped
def test_directory_mutation_after_scandir_snapshot_fails_closed(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
root = tmp_path / "root"
root.mkdir()
(root / "safe.md").write_text("safe")
root_metadata = root.stat()
real_scandir = os.scandir
mutated = False
class MutatingScandir:
def __init__(self, path) -> None:
self._iterator = real_scandir(path)
def __enter__(self):
self._iterator.__enter__()
return self
def __exit__(self, *args):
return self._iterator.__exit__(*args)
def __iter__(self):
return self
def __next__(self):
nonlocal mutated
try:
return next(self._iterator)
except StopIteration:
if not mutated:
(root / "late.md").write_text("late")
os.utime(root, ns=(root_metadata.st_atime_ns, root_metadata.st_mtime_ns + 2_000_000_000))
mutated = True
raise
monkeypatch.setattr(secure_fs.os, "scandir", MutatingScandir)
with pytest.raises(SecurePathError, match=r"directory.*changed|mutation|snapshot"):
discover_secure_files(root, selected=lambda relative: relative.suffix == ".md", max_paths=10)
assert mutated
@pytest.mark.parametrize("listing_order", ["forward", "reversed"])
def test_path_budget_is_consumed_in_name_order(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, listing_order: str
) -> None:
root = tmp_path / "root"
root.mkdir()
for name in ("b.bin", "c.bin", "a.bin"):
(root / name).write_bytes(b"x")
real_scandir = os.scandir
class OrderedScandir:
def __init__(self, path) -> None:
with real_scandir(path) as iterator:
entries = sorted(iterator, key=lambda entry: entry.name)
self._entries = entries if listing_order == "forward" else entries[::-1]
def __enter__(self):
return iter(self._entries)
def __exit__(self, *_args: object) -> None:
return None
monkeypatch.setattr(secure_fs.os, "scandir", OrderedScandir)
# Unused excluded names let the whole listing in, so the budget runs out
# while entries are admitted, not while the directory is listed.
with pytest.raises(SecurePathError) as caught:
discover_secure_files(root, selected=lambda _relative: False, excluded_dirs=("x", "y"), max_paths=2)
assert caught.value.code == "path_count_limit"
assert caught.value.relative_path == "c.bin"
assert caught.value.metadata == {"actual": 3, "limit": 2}
def test_directory_listing_stops_once_the_budget_cannot_fit(tmp_path: Path) -> None:
root = tmp_path / "root"
(root / "nested").mkdir(parents=True)
for index in range(3):
(root / "nested" / f"file-{index}.md").write_text("x")
with pytest.raises(SecurePathError) as caught:
discover_secure_files(root, selected=lambda _relative: False, max_paths=3)
# "nested" used one path; listing a third name in it cannot fit the two left.
assert caught.value.code == "path_count_limit"
assert caught.value.relative_path == "nested"
assert caught.value.metadata == {"actual": 4, "limit": 3}
def _read_descriptor(path: Path, max_bytes: int, **options: object) -> bytes:
descriptor = os.open(path, os.O_RDONLY)
try:
return secure_fs.read_bounded(descriptor, max_bytes, **options)
finally:
os.close(descriptor)
def test_read_bounded_accepts_the_limit_and_refuses_one_byte_more(tmp_path: Path) -> None:
exact = tmp_path / "exact.bin"
exact.write_bytes(b"x" * 70_000)
assert _read_descriptor(exact, 70_000) == b"x" * 70_000
with pytest.raises(SecurePathError) as caught:
_read_descriptor(exact, 69_999, relative_path="nested/exact.bin")
assert caught.value.code == "file_size_limit"
assert caught.value.relative_path == "nested/exact.bin"
assert caught.value.metadata == {"actual_bytes": 70_000, "limit_bytes": 69_999}
with pytest.raises(ValueError, match="non-negative"):
_read_descriptor(exact, -1)
def test_read_bounded_truncates_without_reading_past_the_limit(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
large = tmp_path / "large.bin"
large.write_bytes(bytes(range(256)) * 1_000)
requested: list[int] = []
real_read = os.read
def recording_read(descriptor: int, count: int) -> bytes:
requested.append(count)
return real_read(descriptor, count)
monkeypatch.setattr(secure_fs.os, "read", recording_read)
assert _read_descriptor(large, 100_000, truncate=True) == (bytes(range(256)) * 1_000)[:100_000]
assert sum(requested) == 100_000
assert _read_descriptor(large, 0, truncate=True) == b""
def test_secure_root_reads_a_hard_linked_file_only_when_allowed(tmp_path: Path) -> None:
root = tmp_path / "root"
root.mkdir()
store = tmp_path / "store.js"
store.write_text("console.log('hook')")
os.link(store, root / "hook.js")
with SecureRoot(root) as secure_root:
with pytest.raises(SecurePathError, match=r"hard-linked"):
secure_root.read_bytes(Path("hook.js"), 1024)
content, metadata = secure_root.read_bytes(Path("hook.js"), 1024, allow_hardlinks=True)
assert content == b"console.log('hook')"
assert metadata.st_nlink == 2