Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): isolate per-task staged inputs

Christopher Kevinfe1ea3462d64
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e6c80918..8a89dd5b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -74,6 +74,10 @@ All notable changes to SkillEvaluator are documented in this file.
### Fixed
+- Tier 3 generated tasks now stage only an entry's declared `files`, preventing
+ undeclared fixtures from the shared `evals/files/` directory from appearing
+ in that task's `/workspace/input/`, while preserving copy-all behavior for
+ legacy entries that omit the field.
- Public benchmark cards now omit policy profiles, redact absolute host paths,
and normalize imported internal or retired metadata before publication.
- Previous-version validation now rejects catalog-wide scalar reuse and removal
diff --git a/docs/tier3-live-evaluation.mdx b/docs/tier3-live-evaluation.mdx
index 5c904582..cbeae438 100644
--- a/docs/tier3-live-evaluation.mdx
+++ b/docs/tier3-live-evaluation.mdx
@@ -88,6 +88,12 @@ receive, an `expected_output`, and optional `assertions` the grader checks.
Edit freely — the generated cases are a starting point, and the run is only as
good as its dataset.
+Each eval entry can declare task inputs with a `files` string or list. Paths are
+relative to `evals/` (for example, `"files/case-001/input.txt"`) and only those
+declared paths are staged into that task's `/workspace/input/`. An explicit
+empty list or `null` stages no task input. For backward compatibility, omitting
+the `files` field stages the entire shared `evals/files/` directory.
+
### Run the comparison
```bash title="With-skill vs. without-skill in Docker"
diff --git a/src/skillevaluator/tier3/evals_spec.py b/src/skillevaluator/tier3/evals_spec.py
index ef93fa38..1495d3c9 100644
--- a/src/skillevaluator/tier3/evals_spec.py
+++ b/src/skillevaluator/tier3/evals_spec.py
@@ -200,9 +200,12 @@ grading:
kind="dir",
requirement=Requirement.OPTIONAL,
scope=Scope.HARBOR,
- description="Input files staged into /workspace/input/ inside containers. "
- "Test data, configs, sample documents the skill needs during evaluation.",
- validation_notes=["Any file types. Entire directory is copied into the agent workspace."],
+ description="Shared pool of test data, configs, and sample documents that eval entries can stage "
+ "into /workspace/input/ inside containers.",
+ validation_notes=[
+ "Any file types. An entry with files stages only its declared paths; "
+ "omitting files copies this entire directory for backward compatibility."
+ ],
),
# -- Custom environment --
EntrySpec(
diff --git a/src/skillevaluator/tier3/harbor/adapter.py b/src/skillevaluator/tier3/harbor/adapter.py
index ec1c1550..8cbc951c 100644
--- a/src/skillevaluator/tier3/harbor/adapter.py
+++ b/src/skillevaluator/tier3/harbor/adapter.py
@@ -1469,9 +1469,12 @@ def _stage_task_inputs(
evals_dir: Path,
) -> bool:
input_dir = env_dir / "input"
- if input_files_dir and input_files_dir.exists():
- if input_dir.exists():
- shutil.rmtree(input_dir)
+ if os.path.lexists(input_dir) and (input_dir.is_symlink() or not input_dir.is_dir()):
+ input_dir.unlink()
+ elif input_dir.exists():
+ shutil.rmtree(input_dir)
+
+ if "files" not in entry and input_files_dir and input_files_dir.exists():
copytree_secure(input_files_dir, input_dir, dirs_exist_ok=True)
for ref in _entry_file_refs(entry):
@@ -1704,7 +1707,7 @@ def _write_dockerfile(
)
dockerfile_lines.extend(agent_config_lines)
- if input_files_dir and input_files_dir.exists():
+ if include_input:
dockerfile_lines.append("COPY input/ /workspace/input/")
if include_repo:
dockerfile_lines.append("COPY repo/ /workspace/repo/")
diff --git a/src/skillevaluator/tier3/reference_skills/create-custom-grader/SKILL.md b/src/skillevaluator/tier3/reference_skills/create-custom-grader/SKILL.md
index 3c4eaa18..01bada3d 100644
--- a/src/skillevaluator/tier3/reference_skills/create-custom-grader/SKILL.md
+++ b/src/skillevaluator/tier3/reference_skills/create-custom-grader/SKILL.md
@@ -74,9 +74,13 @@ grader to replace the default evaluator metrics.
2. Map benchmark fields into evaluator inputs.
Use benchmark prompts or prompt variants as `question` entries. Use the
- target skill as `expected_skill`. Put required starter files under
- `evals/files/`. Preserve benchmark-specific rubric text in the entry only
- when the grader needs to read it.
+ target skill as `expected_skill`. Put each case's required starter files
+ under `evals/files/<case-id>/`, and declare
+ `files: ["evals/files/<case-id>"]` on every corresponding eval entry. Do not
+ omit `files` in a multi-case dataset, because omission intentionally stages
+ the entire shared directory for legacy compatibility. Preserve
+ benchmark-specific rubric text in the entry only when the grader needs to
+ read it.
3. Scaffold the evaluator contract.
For generated tasks:
@@ -114,7 +118,8 @@ Python and shell graders run inside the Harbor verifier context. They may read:
- `/logs/agent/trajectory.json` for agent actions and final answer evidence
- `/tests/entry.json` for the eval case metadata
-- `/workspace/input/` for committed fixture files from `evals/files/`
+- `/workspace/input/` for the entry's declared committed fixtures from
+ `evals/files/`
- `/solution/` or other task outputs only when the task environment produces
them
@@ -171,7 +176,8 @@ For a benchmark task with `task.yaml`, `code/`, prompt variants, coverage, and a
rubric:
1. Copy `code/` into `evals/files/<case-id>/`.
-2. Create one or more `evals/evals.json` entries from the prompt variants.
+2. Create one or more `evals/evals.json` entries from the prompt variants, and
+ set `files: ["evals/files/<case-id>"]` on each corresponding entry.
3. Set `expected_skill` to the benchmark's target skill.
4. Implement `evals/grader.py` to inspect the agent trajectory and changed
workspace files.
diff --git a/tests/test_harbor_input_staging.py b/tests/test_harbor_input_staging.py
index 56f68b5b..0c2de145 100644
--- a/tests/test_harbor_input_staging.py
+++ b/tests/test_harbor_input_staging.py
@@ -3,14 +3,17 @@
"""Regression tests for per-entry eval input staging.
-Ports SkillEvaluator 0.7.22 ``0d17f5e`` ("upload staged eval inputs to standard sandboxes")
-into the in-process Tier 3 engine (``tier3/harbor/adapter.py``). The adapter now
-stages both the shared ``evals/files/`` directory and each entry's ``files`` refs
-into the task ``input/`` dir, with traversal protection.
+Ports Skill Evaluator 0.7.22 ``0d17f5e`` ("upload staged eval inputs to standard sandboxes")
+into the in-process Tier 3 engine (``tier3/harbor/adapter.py``). Entries that
+declare ``files`` stage only those refs; entries that omit ``files`` retain the
+legacy shared ``evals/files/`` behavior. All refs retain traversal protection.
"""
from __future__ import annotations
+import json
+import os
+import shutil
from pathlib import Path
import pytest
@@ -19,15 +22,19 @@ from skillevaluator.tier3.harbor.adapter import (
_entry_file_refs,
_resolve_entry_file_ref,
_stage_task_inputs,
+ generate_harbor_tasks,
)
def _make_skill(tmp_path: Path) -> tuple[Path, Path, Path]:
skill = tmp_path / "myskill"
+ (skill / "SKILL.md").parent.mkdir(parents=True)
+ (skill / "SKILL.md").write_text("# Test skill\n")
evals = skill / "evals"
files = evals / "files"
files.mkdir(parents=True)
(files / "global.txt").write_text("global")
+ (files / "unrelated.txt").write_text("unrelated")
data = evals / "data"
data.mkdir()
(data / "case1.txt").write_text("case1")
@@ -43,6 +50,9 @@ class TestEntryFileRefs:
def test_string_is_wrapped(self):
assert _entry_file_refs({"files": "data/case1.txt"}) == ["data/case1.txt"]
+ def test_explicit_null_returns_empty(self):
+ assert _entry_file_refs({"files": None}) == []
+
def test_list_is_passed_through(self):
assert _entry_file_refs({"files": ["a/b.txt", " c/d.txt "]}) == ["a/b.txt", "c/d.txt"]
@@ -52,15 +62,110 @@ class TestEntryFileRefs:
class TestStageTaskInputs:
- def test_stages_global_files_and_entry_refs(self, tmp_path: Path):
+ def test_explicit_files_stage_only_declared_refs(self, tmp_path: Path):
skill, evals, env_dir = _make_skill(tmp_path)
entry = {"id": "t1", "files": ["data/case1.txt"]}
staged = _stage_task_inputs(
env_dir, input_files_dir=evals / "files", entry=entry, source_skill_path=skill, evals_dir=evals
)
assert staged is True
- names = sorted(p.name for p in (env_dir / "input").rglob("*") if p.is_file())
- assert names == ["case1.txt", "global.txt"]
+ paths = sorted(
+ p.relative_to(env_dir / "input").as_posix() for p in (env_dir / "input").rglob("*") if p.is_file()
+ )
+ assert paths == ["data/case1.txt"]
+
+ def test_explicit_file_under_shared_directory_stages_only_that_file(self, tmp_path: Path):
+ skill, evals, env_dir = _make_skill(tmp_path)
+ staged = _stage_task_inputs(
+ env_dir,
+ input_files_dir=evals / "files",
+ entry={"id": "t1", "files": "files/global.txt"},
+ source_skill_path=skill,
+ evals_dir=evals,
+ )
+ assert staged is True
+ paths = sorted(
+ p.relative_to(env_dir / "input").as_posix() for p in (env_dir / "input").rglob("*") if p.is_file()
+ )
+ assert paths == ["global.txt"]
+
+ def test_omitted_files_stages_entire_shared_directory(self, tmp_path: Path):
+ skill, evals, env_dir = _make_skill(tmp_path)
+ staged = _stage_task_inputs(
+ env_dir,
+ input_files_dir=evals / "files",
+ entry={"id": "t1"},
+ source_skill_path=skill,
+ evals_dir=evals,
+ )
+ assert staged is True
+ paths = sorted(
+ p.relative_to(env_dir / "input").as_posix() for p in (env_dir / "input").rglob("*") if p.is_file()
+ )
+ assert paths == ["global.txt", "unrelated.txt"]
+
+ @pytest.mark.parametrize("files", [[], None, "", " "])
+ def test_explicit_empty_files_stages_nothing_and_cleans_stale_input(self, tmp_path: Path, files: object):
+ skill, evals, env_dir = _make_skill(tmp_path)
+ input_dir = env_dir / "input"
+ input_dir.mkdir()
+ (input_dir / "stale.txt").write_text("stale")
+
+ staged = _stage_task_inputs(
+ env_dir,
+ input_files_dir=evals / "files",
+ entry={"id": "t1", "files": files},
+ source_skill_path=skill,
+ evals_dir=evals,
+ )
+
+ assert staged is False
+ assert not input_dir.exists()
+
+ @pytest.mark.parametrize("stale_kind", ["file", "symlink", "fifo"])
+ def test_explicit_empty_files_safely_cleans_non_directory_input(self, tmp_path: Path, stale_kind: str):
+ skill, evals, env_dir = _make_skill(tmp_path)
+ input_path = env_dir / "input"
+ symlink_target = tmp_path / "outside.txt"
+ symlink_target.write_text("keep")
+ if stale_kind == "file":
+ input_path.write_text("stale")
+ elif stale_kind == "symlink":
+ input_path.symlink_to(symlink_target)
+ else:
+ if not hasattr(os, "mkfifo"):
+ pytest.skip("FIFOs are unavailable on this platform")
+ os.mkfifo(input_path)
+
+ staged = _stage_task_inputs(
+ env_dir,
+ input_files_dir=evals / "files",
+ entry={"id": "t1", "files": []},
+ source_skill_path=skill,
+ evals_dir=evals,
+ )
+
+ assert staged is False
+ assert not os.path.lexists(input_path)
+ assert symlink_target.read_text() == "keep"
+
+ def test_explicit_refs_replace_stale_input(self, tmp_path: Path):
+ skill, evals, env_dir = _make_skill(tmp_path)
+ input_dir = env_dir / "input"
+ input_dir.mkdir()
+ (input_dir / "stale.txt").write_text("stale")
+
+ staged = _stage_task_inputs(
+ env_dir,
+ input_files_dir=evals / "files",
+ entry={"id": "t1", "files": ["data/case1.txt"]},
+ source_skill_path=skill,
+ evals_dir=evals,
+ )
+
+ assert staged is True
+ paths = sorted(p.relative_to(input_dir).as_posix() for p in input_dir.rglob("*") if p.is_file())
+ assert paths == ["data/case1.txt"]
def test_no_inputs_returns_false(self, tmp_path: Path):
skill = tmp_path / "myskill"
@@ -74,6 +179,55 @@ class TestStageTaskInputs:
assert staged is False
+def test_generated_tasks_apply_per_entry_input_isolation(tmp_path: Path):
+ skill, evals, _ = _make_skill(tmp_path)
+ entries = [
+ {"id": "legacy", "question": "legacy"},
+ {"id": "selected", "question": "selected", "files": ["data/case1.txt"]},
+ {"id": "empty", "question": "empty", "files": []},
+ {"id": "null", "question": "null", "files": None},
+ ]
+ (evals / "evals.json").write_text(json.dumps(entries))
+
+ task_dirs = generate_harbor_tasks(skill, tmp_path / "generated")
+ tasks = {task.name: task for task in task_dirs}
+
+ def staged_paths(case_id: str) -> list[str]:
+ input_dir = tasks[case_id] / "environment" / "input"
+ if not input_dir.exists():
+ return []
+ return sorted(path.relative_to(input_dir).as_posix() for path in input_dir.rglob("*") if path.is_file())
+
+ assert staged_paths("legacy") == ["global.txt", "unrelated.txt"]
+ assert staged_paths("selected") == ["data/case1.txt"]
+ assert staged_paths("empty") == []
+ assert staged_paths("null") == []
+
+ for case_id in ("legacy", "selected"):
+ dockerfile = (tasks[case_id] / "environment" / "Dockerfile").read_text()
+ assert "COPY input/ /workspace/input/" in dockerfile
+ for case_id in ("empty", "null"):
+ dockerfile = (tasks[case_id] / "environment" / "Dockerfile").read_text()
+ assert "COPY input/ /workspace/input/" not in dockerfile
+
+
[email protected]("base_image", ["", "example.invalid/eval-base:latest"])
+def test_generated_task_copies_explicit_ref_without_shared_files_directory(tmp_path: Path, base_image: str):
+ skill, evals, _ = _make_skill(tmp_path)
+ shutil.rmtree(evals / "files")
+ entries = [{"id": "selected", "question": "selected", "files": "data/case1.txt"}]
+ (evals / "evals.json").write_text(json.dumps(entries))
+
+ task = generate_harbor_tasks(skill, tmp_path / "generated", base_image=base_image)[0]
+
+ input_dir = task / "environment" / "input"
+ assert [path.relative_to(input_dir).as_posix() for path in input_dir.rglob("*") if path.is_file()] == [
+ "data/case1.txt"
+ ]
+ dockerfile = (task / "environment" / "Dockerfile").read_text()
+ assert "COPY input/ /workspace/input/" in dockerfile
+
+
class TestResolveEntryFileRef:
def test_traversal_outside_evals_blocked(self, tmp_path: Path):
skill, evals, _ = _make_skill(tmp_path)