Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): close projection edge cases

Christopher Kevindef5ca1abe74
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 753ea0e0..90326db5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -84,7 +84,11 @@ All notable changes to SkillEvaluator are documented in this file.
environments, and declared inputs continue to load from the source dataset.
Authenticated historical result trees are also excluded after output rotation,
invalid markers fail closed, and late Codex, Cline, Goose, and Qwen
- skill-discovery roots are reset before agent execution.
+ skill-discovery roots are reset before agent execution. Pre-upgrade custom
+ result roots outside `evals/` have no authenticity marker and cannot be
+ distinguished safely from authored runtime content. Move or delete that old
+ content before `--copy-repo` or other full-context evaluation, then rerun with
+ this version if replacement evidence is needed.
- Public benchmark cards now omit policy profiles, redact absolute host paths,
and normalize imported internal or retired metadata before publication.
- Previous-version validation now rejects catalog-wide scalar reuse and removal
diff --git a/docs/tier3-live-evaluation.mdx b/docs/tier3-live-evaluation.mdx
index 5604b9b8..b8218346 100644
--- a/docs/tier3-live-evaluation.mdx
+++ b/docs/tier3-live-evaluation.mdx
@@ -101,7 +101,11 @@ nested, reference, or workspace skill into agent-visible copies. `--copy-repo`
uses the same filtering and excludes generated result roots. Authenticated
historical result roots remain excluded after the configured output location is
changed; a present marker that is copied, stale, or cannot be authenticated
-fails staging closed instead of copying that tree. Runtime assets that
+fails staging closed instead of copying that tree. Custom result roots created
+outside `evals/` by a pre-marker version cannot be identified safely after the
+configured location changes. Move or delete that old content before
+`--copy-repo` or another full-context evaluation, then rerun with the current
+version if replacement evidence is needed. Runtime assets that
the agent needs should live outside skill-owned `evals/`, or be declared as task
inputs with `files`.
diff --git a/src/skillevaluator/tier3/harbor/adapter.py b/src/skillevaluator/tier3/harbor/adapter.py
index 8053edda..9153b1d7 100644
--- a/src/skillevaluator/tier3/harbor/adapter.py
+++ b/src/skillevaluator/tier3/harbor/adapter.py
@@ -27,7 +27,7 @@ import stat
import subprocess
import tempfile
import tomllib
-from collections.abc import Sequence
+from collections.abc import Callable, Iterable, Sequence
from pathlib import Path
from typing import Any
from urllib.parse import unquote
@@ -81,6 +81,8 @@ _REPO_CONTEXT_IGNORE_NAMES = {
}
_REPO_CONTEXT_IGNORE_PARTS = {("evals", "results")}
_REPO_CONTEXT_IGNORE_SUFFIXES = (".pem", ".key", ".p12", ".pfx")
+_NATIVE_SOURCE_IGNORE_NAMES = ("results", "__pycache__", ".git", GENERATED_OUTPUT_MARKER)
+_NATIVE_SOURCE_IGNORE = shutil.ignore_patterns(*_NATIVE_SOURCE_IGNORE_NAMES)
_REPO_CONTEXT_PUBLIC_ENV_SUFFIXES = (".dist", ".example", ".sample", ".template")
_REPO_CONTEXT_SENSITIVE_NAMES = {
".git-credentials",
@@ -291,6 +293,7 @@ def _find_target_manifest_payload(
runtime_projection: bool = False,
excluded_roots: Sequence[Path] = (),
ignored_parts: frozenset[str] = frozenset(),
+ ignored_path_predicate: Callable[[Path], bool] | None = None,
) -> Path | None:
"""Find a renamed file containing the exact evaluated-skill instructions."""
target_manifest = _skill_manifest(target_skill)
@@ -299,6 +302,8 @@ def _find_target_manifest_payload(
target_payload = target_manifest.read_bytes()
runtime_ignore = _runtime_skill_copy_ignore(root, excluded_roots) if runtime_projection else None
for candidate in root.rglob("*"):
+ if ignored_path_predicate is not None and ignored_path_predicate(candidate):
+ continue
relative_parts = candidate.relative_to(root).parts
if ignored_parts.intersection(relative_parts):
continue
@@ -3500,9 +3505,10 @@ def _write_dockerfile(
custom_env_dir = skill_path / "evals" / "environment" if skill_path else None
if custom_env_dir and custom_env_dir.exists():
- staged_custom_env = env_dir / ".skillevaluator-custom-environment"
- copytree_secure(custom_env_dir, staged_custom_env, allowed_root=skill_path)
+ private_custom_env = tempfile.TemporaryDirectory(prefix="skillevaluator-custom-environment-")
+ staged_custom_env = Path(private_custom_env.name) / "environment"
try:
+ copytree_secure(custom_env_dir, staged_custom_env, allowed_root=skill_path)
if not has_skill and skill_path:
_check_custom_environment_does_not_stage_target(staged_custom_env, skill_path)
@@ -3571,7 +3577,7 @@ def _write_dockerfile(
)
return
finally:
- shutil.rmtree(staged_custom_env, ignore_errors=True)
+ private_custom_env.cleanup()
if base_image:
dockerfile_lines = [
@@ -3899,15 +3905,31 @@ def _prepare_native_environment(
)
-def _native_task_dirs(dataset_dir: Path) -> list[Path]:
+def _native_task_dirs(
+ dataset_dir: Path,
+ *,
+ ignore: Callable[[str, list[str]], Iterable[str]] | None = None,
+) -> list[Path]:
"""Return native Harbor task directories from a copied dataset."""
+ children = sorted(dataset_dir.iterdir())
+ ignored = set(ignore(os.fspath(dataset_dir), [path.name for path in children])) if ignore is not None else set()
return [
p
- for p in sorted(dataset_dir.iterdir())
- if p.is_dir() and not p.name.startswith((".", "_")) and (p / "task.toml").exists()
+ for p in children
+ if p.name not in ignored and p.is_dir() and not p.name.startswith((".", "_")) and (p / "task.toml").exists()
]
+def _native_source_path_is_ignored(path: Path, native_dir: Path) -> bool:
+ """Apply the native copy ignore callback to every lexical path component."""
+ parent = native_dir
+ for part in path.relative_to(native_dir).parts:
+ if part in _NATIVE_SOURCE_IGNORE(os.fspath(parent), [part]):
+ return True
+ parent /= part
+ return False
+
+
def _native_entry_id(task_dir: Path) -> str:
task_toml = task_dir / "task.toml"
try:
@@ -4163,10 +4185,8 @@ def _check_native_source_does_not_stage_target(
*,
with_skill: bool,
) -> None:
- ignored_parts = frozenset({"results", "__pycache__", ".git"})
-
def _is_ignored(path: Path) -> bool:
- return bool(ignored_parts.intersection(path.relative_to(native_dir).parts))
+ return _native_source_path_is_ignored(path, native_dir)
target_skill_name = target_skill.name
for skill_md in _iter_skill_manifests(native_dir):
@@ -4191,7 +4211,7 @@ def _check_native_source_does_not_stage_target(
payload = _find_target_manifest_payload(
environment_dir,
target_skill,
- ignored_parts=ignored_parts,
+ ignored_path_predicate=_is_ignored,
)
if payload is not None:
raise ValueError(
@@ -4251,7 +4271,7 @@ def _stage_native_harbor_tasks_into(
_validate_output_roots_outside_evaluator_sources(skill_path, repo_context_exclude_paths)
_check_native_source_does_not_stage_target(native_dir, skill_path, with_skill=with_skill)
entries_by_id = _load_entries_by_id(skill_path)
- source_task_dirs = _native_task_dirs(native_dir)
+ source_task_dirs = _native_task_dirs(native_dir, ignore=_NATIVE_SOURCE_IGNORE)
if not source_task_dirs:
raise ValueError(f"No Harbor task directories with task.toml found in {native_dir}")
validate_case_ids(path.name for path in source_task_dirs)
@@ -4265,7 +4285,7 @@ def _stage_native_harbor_tasks_into(
copytree_secure(
native_dir,
output_dir,
- ignore=shutil.ignore_patterns("results", "__pycache__", ".git", GENERATED_OUTPUT_MARKER),
+ ignore=_NATIVE_SOURCE_IGNORE,
)
output_dir.mkdir(parents=True, exist_ok=True)
task_dirs = _native_task_dirs(output_dir)
diff --git a/tests/test_harbor_runtime_skill_isolation.py b/tests/test_harbor_runtime_skill_isolation.py
index 7bcd72ea..612b2c4b 100644
--- a/tests/test_harbor_runtime_skill_isolation.py
+++ b/tests/test_harbor_runtime_skill_isolation.py
@@ -15,6 +15,7 @@ from pathlib import Path
import pytest
+import skillevaluator.tier3.harbor.adapter as adapter_module
from skillevaluator.tier3.harbor.adapter import (
_collect_all_skill_deps,
_dockerfile_resolved_build_context_sources,
@@ -1322,6 +1323,62 @@ def test_explicit_empty_input_keeps_authored_copy_input_buildable(
assert copy_instruction in (task / "environment" / "Dockerfile").read_text(encoding="utf-8")
+def test_readonly_custom_environment_snapshot_stays_outside_docker_context(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ if os.name != "posix":
+ pytest.skip("POSIX directory modes required")
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ (target / "evals" / "evals.json").write_text(
+ json.dumps([{"id": "case-001", "question": "No fixture.", "files": []}]),
+ encoding="utf-8",
+ )
+ custom_environment = target / "evals" / "environment"
+ snapshot_destinations: list[Path] = []
+ original_copytree = adapter_module.copytree_secure
+
+ def record_custom_snapshot(source: Path, destination: Path, *args: object, **kwargs: object) -> None:
+ if Path(source) == custom_environment:
+ snapshot_destinations.append(Path(destination))
+ original_copytree(source, destination, *args, **kwargs)
+
+ monkeypatch.setattr(adapter_module, "copytree_secure", record_custom_snapshot)
+ authored_input = custom_environment / "input"
+ authored_input.mkdir()
+ (authored_input / "hidden.txt").write_text("UNDECLARED-CUSTOM-INPUT\n", encoding="utf-8")
+ (custom_environment / "Dockerfile").write_text(
+ "FROM python:3.12-slim\nCOPY . /context\n",
+ encoding="utf-8",
+ )
+ sidecar = custom_environment / "sidecar"
+ sidecar.chmod(0o555)
+ custom_environment.chmod(0o555)
+ try:
+ task = generate_harbor_tasks(
+ target,
+ tmp_path / "readonly-custom-environment",
+ custom_dockerfile_mode="preserve",
+ grading_mode="custom_only",
+ )[0]
+ finally:
+ custom_environment.chmod(0o755)
+ sidecar.chmod(0o755)
+
+ environment = task / "environment"
+ assert len(snapshot_destinations) == 1
+ assert not snapshot_destinations[0].absolute().is_relative_to(environment.absolute())
+ assert not any(path.name == ".skillevaluator-custom-environment" for path in environment.rglob("*"))
+ assert "COPY . /context" in (environment / "Dockerfile").read_text(encoding="utf-8")
+ assert (environment / "input").is_dir()
+ assert list((environment / "input").iterdir()) == []
+ assert not (environment / "input" / "hidden.txt").exists()
+ readable_environment = "\n".join(
+ path.read_text(encoding="utf-8", errors="ignore") for path in environment.rglob("*") if path.is_file()
+ )
+ assert "UNDECLARED-CUSTOM-INPUT" not in readable_environment
+
+
def test_native_copy_input_without_fixtures_gets_empty_build_context_directory(tmp_path: Path) -> None:
_, target, _, _ = _write_projection_fixture(tmp_path)
(target / "evals" / "evals.json").write_text(
@@ -1670,6 +1727,49 @@ def test_native_baseline_ignores_target_payload_in_unstaged_results(tmp_path: Pa
assert not (task / "environment" / "results").exists()
+def test_native_task_discovery_ignores_unstaged_root_results(tmp_path: Path) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ archived_task = target / "evals" / "harbor" / "results"
+ archived_task.mkdir()
+ (archived_task / "task.toml").write_text(
+ 'schema_version = "1.3"\n\n[task]\nname = "nvidia/archived"\n\n'
+ '[metadata]\nentry_id = "archived"\n\n[environment]\nworkdir = "relative"\n',
+ encoding="utf-8",
+ )
+
+ tasks = stage_native_harbor_tasks(target, tmp_path / "native-ignored-root-results")
+
+ assert [task.name for task in tasks] == ["case-001"]
+ assert not (tmp_path / "native-ignored-root-results" / "results").exists()
+
+
+def test_native_contamination_scan_uses_shared_ignore_callback(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ ignored = target / "evals" / "harbor" / "case-001" / "environment" / "RESULTS"
+ ignored.mkdir(parents=True)
+ (ignored / "archived.txt").write_bytes((target / "SKILL.md").read_bytes())
+
+ ignored_casefolded = frozenset(name.casefold() for name in adapter_module._NATIVE_SOURCE_IGNORE_NAMES)
+
+ def _casefolding_ignore(_directory: str, names: list[str]) -> set[str]:
+ return {name for name in names if name.casefold() in ignored_casefolded}
+
+ monkeypatch.setattr(adapter_module, "_NATIVE_SOURCE_IGNORE", _casefolding_ignore)
+
+ task = stage_native_harbor_tasks(
+ target,
+ tmp_path / "native-shared-ignore-callback",
+ with_skill=False,
+ )[0]
+
+ assert not (task / "environment" / "RESULTS").exists()
+
+
@pytest.mark.parametrize(
"copy_instruction",
[