Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(tier3): close remaining fixture isolation gaps
Christopher Kevin7fb40e5da2d9
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ff23ffeb..753ea0e0 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -82,6 +82,9 @@ All notable changes to SkillEvaluator are documented in this file.from every staged skill package, including sanitized `--copy-repo` contexts,while graders, native tasks, customenvironments, and declared inputs continue to load from the source dataset.+ Authenticated historical result trees are also excluded after output rotation,+ invalid markers fail closed, and late Codex, Cline, Goose, and Qwen+ skill-discovery roots are reset before agent execution.- Public benchmark cards now omit policy profiles, redact absolute host paths,and normalize imported internal or retired metadata before publication.- Previous-version validation now rejects catalog-wide scalar reuse and removaldiff --git a/docs/tier3-live-evaluation.mdx b/docs/tier3-live-evaluation.mdxindex c7e834ea..5604b9b8 100644--- a/docs/tier3-live-evaluation.mdx+++ b/docs/tier3-live-evaluation.mdx@@ -98,7 +98,10 @@ An `evals/` directory directly owned by a skill package is evaluator-only. Tier3 reads datasets, graders, native Harbor tasks, and custom environments from thesource tree, but does not install those directories from the target or anynested, reference, or workspace skill into agent-visible copies. `--copy-repo`-uses the same filtering and excludes generated result roots. Runtime assets that+uses the same filtering and excludes generated result roots. Authenticated+historical result roots remain excluded after the configured output location is+changed; a present marker that is copied, stale, or cannot be authenticated+fails staging closed instead of copying that tree. Runtime assets thatthe agent needs should live outside skill-owned `evals/`, or be declared as taskinputs with `files`.diff --git a/src/skillevaluator/tier3/harbor/adapter.py b/src/skillevaluator/tier3/harbor/adapter.pyindex 6e90a157..8053edda 100644--- a/src/skillevaluator/tier3/harbor/adapter.py+++ b/src/skillevaluator/tier3/harbor/adapter.py@@ -323,14 +323,16 @@ def _find_target_manifest_payload(return None+def _casefold_contained(candidate: Path, root: Path) -> bool:+ """Return whether *candidate* is lexically below *root*, ignoring path casing."""+ candidate_parts = tuple(part.casefold() for part in candidate.parts)+ root_parts = tuple(part.casefold() for part in root.parts)+ return len(candidate_parts) >= len(root_parts) and candidate_parts[: len(root_parts)] == root_parts++def _path_is_excluded(path: Path, excluded_roots: Sequence[Path]) -> bool:"""Check lexical and resolved containment using cross-platform path casing."""- def _casefold_contained(candidate: Path, root: Path) -> bool:- candidate_parts = tuple(part.casefold() for part in candidate.parts)- root_parts = tuple(part.casefold() for part in root.parts)- return len(candidate_parts) >= len(root_parts) and candidate_parts[: len(root_parts)] == root_parts-for excluded_root in excluded_roots:if _casefold_contained(path.absolute(), excluded_root.absolute()):return True@@ -378,18 +380,75 @@ def _path_is_canonically_contained(path: Path, root: Path) -> bool:return True+def _authenticated_generated_output_ancestor(+ path: Path,+ root: Path,+ authenticated_roots: set[Path] | None = None,+) -> Path | None:+ """Find an authenticated generated-output boundary or reject an unsafe marker."""+ lexical_path = path.absolute()+ for authenticated_root in authenticated_roots or ():+ if _casefold_contained(lexical_path, authenticated_root.absolute()):+ return authenticated_root++ try:+ metadata = path.lstat()+ except FileNotFoundError:+ current = path.parent+ except OSError as exc:+ raise ValueError(f"Cannot inspect potential generated output path: {path}") from exc+ else:+ current = (+ path if stat.S_ISDIR(metadata.st_mode) and not _path_is_link_or_reparse(path, metadata) else path.parent+ )++ lexical_root = root.absolute()+ try:+ resolved_root = root.resolve()+ except (OSError, RuntimeError):+ resolved_root = lexical_root++ def _inside_root(candidate: Path) -> bool:+ absolute_candidate = candidate.absolute()+ if _casefold_contained(absolute_candidate, lexical_root) or _casefold_contained(+ absolute_candidate, resolved_root+ ):+ return True+ try:+ return _casefold_contained(candidate.resolve(), resolved_root)+ except (OSError, RuntimeError):+ return False++ while _inside_root(current):+ marker = current / GENERATED_OUTPUT_MARKER+ if os.path.lexists(marker):+ if _path_is_link_or_reparse(current):+ raise ValueError(f"Generated output marker is under an unsafe linked directory: {current}")+ try:+ authenticated = is_generated_output_root(current)+ except (OSError, ValueError) as exc:+ raise ValueError(f"Generated output marker is invalid or cannot be authenticated: {current}") from exc+ if not authenticated:+ raise ValueError(f"Generated output marker is invalid or cannot be authenticated: {current}")+ if authenticated_roots is not None:+ authenticated_roots.add(current)+ return current+ if _paths_equivalent(current, root):+ break+ parent = current.parent+ if parent == current:+ break+ current = parent+ return None++def _inside_declared_generated_dataset(path: Path, declared_roots: Sequence[Path]) -> bool:"""Return whether *path* is below a staged dataset marker in a declared output root."""for declared_root in declared_roots:if not _path_is_excluded(path, (declared_root,)):continue- current = path if path.is_dir() else path.parent- while _path_is_excluded(current, (declared_root,)):- if is_generated_output_root(current):- return True- if _paths_equivalent(current, declared_root):- break- current = current.parent+ if _authenticated_generated_output_ancestor(path, declared_root) is not None:+ return Truereturn False@@ -535,6 +594,16 @@ def _runtime_skill_copy_ignore(skill_root: Path, excluded_roots: Sequence[Path]"""Build a root-aware ignore callback for an agent-visible skill copy."""resolved_root = skill_root.resolve()excluded_roots = _skill_scoped_excluded_roots(skill_root, excluded_roots)+ authenticated_output_roots: set[Path] = set()+ if (+ _authenticated_generated_output_ancestor(+ skill_root,+ skill_root,+ authenticated_output_roots,+ )+ is not None+ ):+ raise ValueError(f"Runtime skill source must not be a generated output root: {skill_root}")def _ignore(directory: str, contents: list[str]) -> list[str]:current = Path(directory)@@ -543,7 +612,16 @@ def _runtime_skill_copy_ignore(skill_root: Path, excluded_roots: Sequence[Path]ignored.update(name for name in contents if name.casefold() == "evals")for name in contents:candidate = current / name- if _is_skill_evals_path(candidate, skill_root) or _path_is_excluded(candidate, excluded_roots):+ if (+ _is_skill_evals_path(candidate, skill_root)+ or _path_is_excluded(candidate, excluded_roots)+ or _authenticated_generated_output_ancestor(+ candidate,+ skill_root,+ authenticated_output_roots,+ )+ is not None+ ):ignored.add(name)return sorted(ignored)@@ -735,17 +813,27 @@ def _staged_relative_link_path(skill_name: str, raw_target: str) -> Path | None:"skills",".agents",".claude",+ ".cline",".codex",".config",".cursor",".gemini",".opencode",+ ".qwen",}:return Nonereturn rel-def _repo_context_ignore_file(path: Path, root: Path) -> bool:+def _repo_context_ignore_file(+ path: Path,+ root: Path,+ authenticated_output_roots: set[Path] | None = None,+) -> bool:+ if authenticated_output_roots is None:+ authenticated_output_roots = set()+ if _authenticated_generated_output_ancestor(path, root, authenticated_output_roots) is not None:+ return Truetry:path.resolve().relative_to(root.resolve())except ValueError:@@ -821,11 +909,19 @@ def _git_context_files(root: Path) -> list[Path]:return [root / line for line in result.stdout.splitlines() if line.strip()]-def _iter_repo_context_files(root: Path) -> list[Path]:+def _iter_repo_context_files(root: Path, authenticated_output_roots: set[Path]) -> list[Path]:git_files = _git_context_files(root)if git_files:- return sorted(path for path in git_files if path.is_file())- return sorted(path for path in root.rglob("*") if path.is_file() and not _repo_context_ignore_file(path, root))+ return sorted(+ path+ for path in git_files+ if path.is_file() and not _repo_context_ignore_file(path, root, authenticated_output_roots)+ )+ return sorted(+ path+ for path in root.rglob("*")+ if path.is_file() and not _repo_context_ignore_file(path, root, authenticated_output_roots)+ )def _stage_repo_context(@@ -843,6 +939,16 @@ def _stage_repo_context(source_skill_path = source_skill_path.resolve()skill_md = _skill_manifest(source_skill_path)repo_root = _repo_context_root(source_skill_path)+ authenticated_output_roots: set[Path] = set()+ if (+ _authenticated_generated_output_ancestor(+ repo_root,+ repo_root,+ authenticated_output_roots,+ )+ is not None+ ):+ raise ValueError(f"Repo context root must not be a generated output root: {repo_root}")resolved_excluded_roots: list[Path] = []for excluded_root in excluded_roots:@@ -863,10 +969,10 @@ def _stage_repo_context(staged: list[dict[str, str]] = []if mode == "full":- for src in _iter_repo_context_files(repo_root):+ for src in _iter_repo_context_files(repo_root, authenticated_output_roots):if _is_excluded_output(src):continue- if _repo_context_ignore_file(src, repo_root):+ if _repo_context_ignore_file(src, repo_root, authenticated_output_roots):continueif exclude_source_skill and _is_relative_to(src, source_skill_path):continue@@ -901,8 +1007,14 @@ def _stage_repo_context(if _is_excluded_output(target_path) or _path_is_excluded(lexical_target_path, resolved_excluded_roots):logger.warning("Skipping SKILL.md link into generated output: %s", raw_target)continue- if _repo_context_ignore_file(lexical_target_path, repo_root) or _repo_context_ignore_file(- target_path, repo_root+ if _repo_context_ignore_file(+ lexical_target_path,+ repo_root,+ authenticated_output_roots,+ ) or _repo_context_ignore_file(+ target_path,+ repo_root,+ authenticated_output_roots,):logger.warning("Skipping ignored SKILL.md repo link: %s", raw_target)continue@@ -2717,22 +2829,29 @@ _AGENT_SKILL_PATHS = []_RUNTIME_SKILL_DIRS = ("/etc/codex/skills",+ "/tmp/codex-home/skills","/workspace/skills","/workspace/.agents/skills","/workspace/.claude/commands","/workspace/.claude/skills",+ "/workspace/.cline/skills","/workspace/.codex/skills",+ "/workspace/.config/goose/skills","/workspace/.config/opencode/skills","/workspace/.cursor/skills","/workspace/.gemini/skills","/workspace/.gemini/extensions","/workspace/.opencode/skills",+ "/workspace/.qwen/skills","/root/.claude/commands","/root/.claude/skills",+ "/root/.cline/skills","/root/.agents/skills","/root/.codex/skills",+ "/root/.config/goose/skills","/root/.config/opencode/skills","/root/.gemini/extensions",+ "/root/.qwen/skills","/logs/agent/sessions/skills","/tmp/agent-home/sessions/skills",)@@ -2740,12 +2859,15 @@ _PROJECT_SKILL_RELATIVE_DIRS = (".agents/skills",".claude/commands",".claude/skills",+ ".cline/skills",".codex/skills",+ ".config/goose/skills",".config/opencode/skills",".cursor/skills",".gemini/skills",".gemini/extensions",".opencode/skills",+ ".qwen/skills",)_RUNTIME_DISCOVERY_ENV_NAMES = frozenset({@@ -2842,11 +2964,12 @@ _RUNTIME_PROJECTION_OVERLAP_PREFLIGHT = ("/workspace/input|/workspace/input/*|/workspace/repo|/workspace/repo/*|""/workspace/skills|/workspace/skills/*|""*/.agents/skills|*/.agents/skills/*|*/.claude/commands|*/.claude/commands/*|"- "*/.claude/skills|*/.claude/skills/*|"- "*/.codex/skills|*/.codex/skills/*|*/.config/opencode/skills|*/.config/opencode/skills/*|"+ "*/.claude/skills|*/.claude/skills/*|*/.cline/skills|*/.cline/skills/*|"+ "*/.codex/skills|*/.codex/skills/*|*/.config/goose/skills|*/.config/goose/skills/*|"+ "*/.config/opencode/skills|*/.config/opencode/skills/*|""*/.cursor/skills|*/.cursor/skills/*|*/.gemini/skills|*/.gemini/skills/*|""*/.gemini/extensions|*/.gemini/extensions/*|"- "*/.opencode/skills|*/.opencode/skills/*) "+ "*/.opencode/skills|*/.opencode/skills/*|*/.qwen/skills|*/.qwen/skills/*) ""echo 'SkillEvaluator agent workdir/home overlaps an evaluator-controlled runtime projection' >&2; "'exit 78;; esac; }; check_control_path() { value=$1; check_path "$value"; ''if [ -d "$value" ]; then canonical=$(CDPATH= cd -P -- "$value" 2>/dev/null && /bin/pwd -P) || exit 78; '@@ -2899,11 +3022,14 @@ def _validated_agent_workdir(agent_workdir: str | None) -> str | None:discovery_parts = ((".agents", "skills"),(".claude", "skills"),+ (".cline", "skills"),(".codex", "skills"),+ (".config", "goose", "skills"),(".config", "opencode", "skills"),(".cursor", "skills"),(".gemini", "skills"),(".opencode", "skills"),+ (".qwen", "skills"),)if any(parts[index : index + len(candidate)] == candidatediff --git a/tests/test_harbor_runtime_skill_isolation.py b/tests/test_harbor_runtime_skill_isolation.pyindex 0c0c3fac..7bcd72ea 100644--- a/tests/test_harbor_runtime_skill_isolation.py+++ b/tests/test_harbor_runtime_skill_isolation.py@@ -9,6 +9,8 @@ import jsonimport osimport shutilimport subprocess+import sys+import tempfilefrom pathlib import Pathimport pytest@@ -880,6 +882,117 @@ def test_marked_prior_run_does_not_block_declared_results_root(tmp_path: Path) -validate_results_root_location(target, results_root)[email protected]("copy_repo", [False, True])[email protected]("stager_name", ["generated", "native"])+def test_rotated_generated_output_is_excluded_from_runtime_skill_and_repo(+ tmp_path: Path,+ copy_repo: bool,+ stager_name: str,+) -> None:+ repo, target, _, _ = _write_projection_fixture(tmp_path)+ (target / "evals" / "files" / "selected.txt").write_text("OLD-CASE-SECRET\n", encoding="utf-8")+ old_output_root = target / "archived-output"+ old_dataset = old_output_root / "dataset"+ generate_harbor_tasks(+ target,+ old_dataset,+ repo_context_exclude_paths=(old_output_root,),+ )+ assert (old_dataset / "case-001" / "environment" / "input" / "selected.txt").is_file()+ older_output_root = target / "older-output"+ older_dataset = older_output_root / "dataset"+ generate_harbor_tasks(+ target,+ older_dataset,+ repo_context_exclude_paths=(older_output_root,),+ )++ (target / "evals" / "evals.json").write_text(+ json.dumps([{"id": "case-001", "question": "Do not use old fixtures.", "files": []}]),+ encoding="utf-8",+ )+ if stager_name == "native":+ _write_minimal_native_task(target)+ stager = stage_native_harbor_tasks+ else:+ stager = generate_harbor_tasks++ task = stager(target, tmp_path / f"current-{stager_name}-{copy_repo}", copy_repo=copy_repo)[0]+ environment = task / "environment"+ for copied_relative in (Path("archived-output") / "dataset", Path("older-output") / "dataset"):+ assert not (environment / "skills" / target.name / copied_relative).exists()+ if copy_repo:+ assert not (environment / "repo" / target.relative_to(repo) / copied_relative).exists()+ readable = "\n".join(+ path.read_text(encoding="utf-8", errors="ignore") for path in environment.rglob("*") if path.is_file()+ )+ assert "OLD-CASE-SECRET" not in readable++[email protected](sys.platform != "darwin", reason="requires the macOS /var root alias")+def test_rotated_output_is_excluded_through_macos_platform_root_alias() -> None:+ with tempfile.TemporaryDirectory(prefix="issue29-platform-alias-") as temporary:+ root = Path(temporary)+ if root.absolute() == root.resolve():+ pytest.skip("temporary directory does not use a platform root alias")+ _, target, _, _ = _write_projection_fixture(root)+ selected = target / "evals" / "files" / "selected.txt"+ selected.write_text("PLATFORM-ALIAS-SECRET\n", encoding="utf-8")+ old_root = target / "archived-output"+ generate_harbor_tasks(+ target,+ old_root / "dataset",+ repo_context_exclude_paths=(old_root,),+ )+ (target / "evals" / "evals.json").write_text(+ json.dumps([{"id": "case-001", "question": "No old fixtures.", "files": []}]),+ encoding="utf-8",+ )++ task = generate_harbor_tasks(target, root / "current", copy_repo=True)[0]++ readable = "\n".join(+ path.read_text(encoding="utf-8", errors="ignore")+ for path in (task / "environment").rglob("*")+ if path.is_file()+ )+ assert "PLATFORM-ALIAS-SECRET" not in readable++[email protected]("marker_state", ["copied", "rotated-key"])+def test_unverifiable_historical_generated_output_fails_closed(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ marker_state: str,+) -> None:+ _, target, _, _ = _write_projection_fixture(tmp_path)+ old_output_root = target / "archived-output"+ old_dataset = old_output_root / "dataset"+ generate_harbor_tasks(+ target,+ old_dataset,+ repo_context_exclude_paths=(old_output_root,),+ )++ if marker_state == "copied":+ shutil.copytree(old_dataset, target / "copied-output")+ else:+ monkeypatch.setenv(+ "SKILLEVALUATOR_OUTPUT_PROVENANCE_KEY_FILE",+ str(tmp_path / ".rotated-state" / "output-provenance.key"),+ )++ (target / "evals" / "evals.json").write_text(+ json.dumps([{"id": "case-001", "question": "Do not use old fixtures.", "files": []}]),+ encoding="utf-8",+ )+ current = tmp_path / f"current-{marker_state}"+ with pytest.raises(ValueError, match=r"marker.*(?:invalid|authenticated|verified)"):+ generate_harbor_tasks(target, current, copy_repo=True)++ assert not current.exists()++def test_declared_in_skill_output_recovers_after_partial_generation_failure(tmp_path: Path) -> None:_, target, _, _ = _write_projection_fixture(tmp_path)(target / "evals" / "evals.json").write_text(@@ -1294,11 +1407,14 @@ def test_generated_baseline_rejects_renamed_target_manifest_payload(tmp_path: Pa["/workspace/.agents/skills","/workspace/.claude/skills",+ "/workspace/.cline/skills","/workspace/.codex/skills",+ "/workspace/.config/goose/skills","/workspace/.config/opencode/skills","/workspace/.cursor/skills","/workspace/.gemini/skills","/workspace/.opencode/skills",+ "/workspace/.qwen/skills",],)def test_final_projection_clears_authored_project_skill_root(@@ -1366,6 +1482,10 @@ def test_final_projection_clears_effective_workdir_ancestors_and_home(tmp_path:assert 'BASH_ENV=""' in dockerfileassert 'CLAUDE_CODE_DISABLE_POLICY_SKILLS="1"' in dockerfileassert "/etc/codex/skills" in dockerfile+ assert "/tmp/codex-home/skills" in dockerfile+ assert ".config/goose/skills" in dockerfile+ assert ".cline/skills" in dockerfile+ assert ".qwen/skills" in dockerfileassert "WORKDIR /opt/task/subdir" in dockerfile[final_projection:]assert dockerfile.rindex("current=$(/bin/pwd -P)") > dockerfile.rindex("COPY input/ /workspace/input/")task_toml = (task / "task.toml").read_text(encoding="utf-8")@@ -1390,6 +1510,9 @@ def test_generated_tasks_reject_unsafe_agent_workdir(tmp_path: Path, agent_workd"/workspace/repo/subdir","/workspace/skills","/opt/project/.claude/skills/hidden",+ "/opt/project/.cline/skills/hidden",+ "/opt/project/.config/goose/skills/hidden",+ "/opt/project/.qwen/skills/hidden",],)def test_generated_tasks_reject_workdir_inside_runtime_projection(tmp_path: Path, agent_workdir: str) -> None:@@ -2278,6 +2401,9 @@ def test_linked_repo_projection_rejects_reserved_workspace_input([("skills/target-skill/SKILL.md", "../../skills/target-skill/SKILL.md"),(".claude/skills/hidden/SKILL.md", "../../.claude/skills/hidden/SKILL.md"),+ (".cline/skills/hidden/SKILL.md", "../../.cline/skills/hidden/SKILL.md"),+ (".config/goose/skills/hidden/SKILL.md", "../../.config/goose/skills/hidden/SKILL.md"),+ (".qwen/skills/hidden/SKILL.md", "../../.qwen/skills/hidden/SKILL.md"),],)def test_linked_repo_cannot_overlay_agent_discovery_roots(