Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): isolate evaluator inputs from runtimes

Christopher Kevin33eb89577837
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 90326db5..a64e3ff9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -88,7 +88,21 @@ All notable changes to SkillEvaluator are documented in this file.
result roots outside `evals/` have no authenticity marker and cannot be
distinguished safely from authored runtime content. Move or delete that old
content before `--copy-repo` or other full-context evaluation, then rerun with
- this version if replacement evidence is needed.
+ this version if replacement evidence is needed. Explicit task inputs cannot
+ select evaluator-owned datasets, configuration, graders, tests, native tasks,
+ environments, or results. Every agent and baseline arm now reads from one
+ private, selective evaluator snapshot containing the active control files,
+ task-source data, consumed fixtures and grader, and the complete authored
+ custom environment. Legacy omitted-file entries retain the full shared files
+ corpus. Unrelated evaluator subtrees and generated results stay outside the
+ snapshot. MCP configuration and completed-run artifacts are
+ read through bounded descriptor-anchored roots; on Windows, selected file
+ handles deny concurrent writes and deletes while live. Historical unmarked
+ runs created before canonical run-level `result.json` remain discoverable only
+ when their stable configuration and summaries satisfy the complete historical
+ schema. Pre-status scored summaries remain consumable, coherent status-era
+ failures remain visible without contributing scores, and marked current
+ partial runs continue to fail closed.
- Public benchmark cards now omit policy profiles, redact absolute host paths,
and normalize imported internal or retired metadata before publication.
- Previous-version validation now rejects catalog-wide scalar reuse and removal
diff --git a/docs/reports.mdx b/docs/reports.mdx
index a61f6173..571be74e 100644
--- a/docs/reports.mdx
+++ b/docs/reports.mdx
@@ -147,8 +147,17 @@ Each run creates a directory named by its run ID (`YYYYMMDD_HHMMSS`), and a
Read commands (`view`, `compare`, `create-eval-dataset --refine`) honor the
same precedence, then fall back to the legacy `<skill>/evals/results/`
location — so old runs stay visible after you configure
- `SKILLEVALUATOR_RESULTS_DIR`. The results tree is generated output: don't
- commit it.
+ `SKILLEVALUATOR_RESULTS_DIR`. Current timestamped runs are selected only when
+ stable regular `run_config.json` and matching atomic `result.json` artifacts
+ are present. Unmarked runs created before run-level `result.json` remain
+ readable only when `run_config.json` has a recognized historical task source
+ and complete agent metadata, and every configured agent has one stable,
+ identity-matched summary satisfying the full historical score, dimension,
+ pass-at-k, and completion schema. Authenticated summaries from before status
+ fields are consumed as successful; coherent status-era failed summaries stay
+ visible without contributing scores. Any marker presence prevents an
+ incomplete current run from being interpreted as legacy output. The results
+ tree is generated output: don't commit it.
</Note>
## Reading a live-eval report
diff --git a/docs/tier3-live-evaluation.mdx b/docs/tier3-live-evaluation.mdx
index b8218346..724248ed 100644
--- a/docs/tier3-live-evaluation.mdx
+++ b/docs/tier3-live-evaluation.mdx
@@ -95,8 +95,18 @@ empty list or `null` stages no task input. For backward compatibility, omitting
the `files` field stages the entire shared `evals/files/` directory.
An `evals/` directory directly owned by a skill package is evaluator-only. Tier
-3 reads datasets, graders, native Harbor tasks, and custom environments from the
-source tree, but does not install those directories from the target or any
+3 resolves the selected task source once and copies only the evaluator inputs it
+can consume into one private temporary snapshot. That projection includes the
+active dataset and configuration, selected fixtures, the first consumed grader,
+the complete authored custom environment, and native tasks when selected.
+Legacy entries that omit `files` retain the complete shared `evals/files/`
+corpus. Unrelated evaluator subtrees and generated results are not copied. Every
+agent and baseline arm reads the same snapshot, preventing a concurrent source
+replacement from mixing dataset, fixture, grader, custom-environment, or
+native-task identities. The snapshot is removed when the run returns and
+requires temporary space only for the selected compatibility scope.
+
+Tier 3 does not install evaluator-owned directories from the target or any
nested, reference, or workspace skill into agent-visible copies. `--copy-repo`
uses the same filtering and excludes generated result roots. Authenticated
historical result roots remain excluded after the configured output location is
@@ -105,9 +115,20 @@ fails staging closed instead of copying that tree. Custom result roots created
outside `evals/` by a pre-marker version cannot be identified safely after the
configured location changes. Move or delete that old content before
`--copy-repo` or another full-context evaluation, then rerun with the current
-version if replacement evidence is needed. Runtime assets that
-the agent needs should live outside skill-owned `evals/`, or be declared as task
-inputs with `files`.
+version if replacement evidence is needed. Runtime assets that the agent needs
+should live outside skill-owned `evals/`, under `evals/files/`
+(preferred), or under another non-control subtree such as `evals/data/` and be
+declared as task inputs with `files`. A `files` declaration cannot select
+evaluator control material: dataset/configuration files, graders, developer
+eval guidance, benchmark conversion reports, native Harbor tasks, custom
+environments, evaluator tests, or generated results.
+
+MCP declarations are read only from a bounded, stable, single-link regular
+`evals/environment/mcp_servers.toml` through a descriptor-anchored evaluator
+root. Linked configuration files or parents, hardlinks, reparse points, and
+files replaced during the read fail task generation before output mutation. On
+Windows, the selected-file handle also denies concurrent writes and deletion
+for the read.
`evals/environment/input/` is reserved and is not copied into generated tasks.
Directory-level custom Docker `COPY input/` remains buildable for an empty entry
diff --git a/src/skillevaluator/evaluation/tier3_report.py b/src/skillevaluator/evaluation/tier3_report.py
index 0956586a..672247ed 100644
--- a/src/skillevaluator/evaluation/tier3_report.py
+++ b/src/skillevaluator/evaluation/tier3_report.py
@@ -307,7 +307,12 @@ def agent_eval_result_from_directory(
)
run_dir = run_dir.expanduser().resolve()
- agents = load_agent_data(run_dir)
+ from skillevaluator.tier3.results_location import is_legacy_completed_run_dir
+
+ agents = load_agent_data(
+ run_dir,
+ allow_legacy_missing_status=is_legacy_completed_run_dir(run_dir),
+ )
if not agents:
return None
diff --git a/src/skillevaluator/tier3/commands.py b/src/skillevaluator/tier3/commands.py
index 35eeb962..54f5ac41 100644
--- a/src/skillevaluator/tier3/commands.py
+++ b/src/skillevaluator/tier3/commands.py
@@ -49,6 +49,8 @@ from skillevaluator.tier3.harbor.runner import (
)
from skillevaluator.tier3.harbor.secure_copy import copytree_secure
from skillevaluator.tier3.results_location import (
+ _run_timestamp,
+ is_legacy_completed_run_dir,
iter_candidate_results_roots,
ordered_run_directories,
resolve_latest_results,
@@ -944,6 +946,7 @@ def compare_results(skill_path: Path, *, results_dir: Path | None = None) -> int
root_without: dict[str, dict[str, float]] = {}
root_meta: dict[str, dict[str, Any]] = {}
for ts_dir in ordered_run_directories(candidate_root):
+ allow_missing_status = _run_timestamp(ts_dir.name) is None or is_legacy_completed_run_dir(ts_dir)
try:
agent_dirs = sorted(ts_dir.iterdir())
except OSError:
@@ -963,7 +966,7 @@ def compare_results(skill_path: Path, *, results_dir: Path | None = None) -> int
data = json.loads(summary.read_text(encoding="utf-8"))
except (ValueError, OSError):
continue
- scores = _summary_scores(data)
+ scores = _summary_scores(data, allow_missing_status=allow_missing_status)
if scores:
root_with[agent_name] = scores
root_meta[agent_name] = {
@@ -974,7 +977,10 @@ def compare_results(skill_path: Path, *, results_dir: Path | None = None) -> int
wo_summary = agent_dir / "without-skill" / "summary.json"
if wo_summary.exists():
try:
- wo_scores = _summary_scores(json.loads(wo_summary.read_text(encoding="utf-8")))
+ wo_scores = _summary_scores(
+ json.loads(wo_summary.read_text(encoding="utf-8")),
+ allow_missing_status=allow_missing_status,
+ )
if wo_scores:
root_without[agent_name] = wo_scores
except (ValueError, OSError):
@@ -1040,10 +1046,11 @@ def compare_results(skill_path: Path, *, results_dir: Path | None = None) -> int
return 0
-def _summary_scores(data: dict[str, Any]) -> dict[str, float]:
+def _summary_scores(data: dict[str, Any], *, allow_missing_status: bool = False) -> dict[str, float]:
if not isinstance(data, dict):
return {}
- if data.get("execution_status") != "succeeded":
+ status = data.get("execution_status")
+ if status != "succeeded" and not (allow_missing_status and status is None):
return {}
scores: dict[str, float] = {}
raw_scores = data.get("scores", data)
diff --git a/src/skillevaluator/tier3/harbor/adapter.py b/src/skillevaluator/tier3/harbor/adapter.py
index af82d626..887d0df8 100644
--- a/src/skillevaluator/tier3/harbor/adapter.py
+++ b/src/skillevaluator/tier3/harbor/adapter.py
@@ -27,7 +27,9 @@ import stat
import subprocess
import tempfile
import tomllib
-from collections.abc import Callable, Iterable, Sequence
+from collections.abc import Callable, Iterable, Iterator, Sequence
+from contextlib import contextmanager
+from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.parse import unquote
@@ -44,6 +46,7 @@ from skillevaluator.tier3.output_provenance import (
)
from skillevaluator.tier3.toml_utils import toml_quote
from skillevaluator.utils.process_environment import child_process_env
+from skillevaluator.utils.secure_fs import SecurePathError, SecureRoot
logger = logging.getLogger(__name__)
@@ -52,6 +55,32 @@ _EVAL_CORE_DIR = Path(__file__).resolve().parent.parent / "eval_core"
_BASE_IMAGE_PREFIX = "skillevaluator-base"
_MAX_REPO_CONTEXT_FILE_BYTES = 10 * 1024 * 1024
_MAX_REPO_CONTEXT_TOTAL_BYTES = 200 * 1024 * 1024
+_MAX_EVALUATOR_FILE_BYTES = 64 * 1024 * 1024
+_MAX_EVALS_CONFIG_BYTES = 4 * 1024 * 1024
+_MAX_MCP_CONFIG_BYTES = 1024 * 1024
+_EVALUATOR_DATASET_FILENAMES = (
+ "evals.json",
+ "evals.jsonl",
+ "evals.yaml",
+ "evals.yml",
+ "dataset.json",
+ "dataset.jsonl",
+ "dataset.yaml",
+ "dataset.yml",
+)
+_EVALUATOR_ONLY_TASK_INPUT_FILES = frozenset(
+ {
+ *(name.casefold() for name in _EVALUATOR_DATASET_FILENAMES),
+ "benchmark_" + "conversion_report.md",
+ "config.yaml",
+ "config.yml",
+ "eval.md",
+ "grader.py",
+ "grader.sh",
+ GENERATED_OUTPUT_MARKER.casefold(),
+ }
+)
+_EVALUATOR_ONLY_TASK_INPUT_ROOTS = frozenset({"environment", "harbor", "results", "tests"})
_REPO_CONTEXT_IGNORE_NAMES = {
".git",
".hg",
@@ -1157,6 +1186,7 @@ def build_eval_base_image(
reference_skills_dir: Path | None = None,
*,
workspace_skill_paths: list[Path] | None = None,
+ evaluator_skill_path: Path | None = None,
excluded_roots: Sequence[Path] = (),
force_rebuild: bool = False,
action_out: list[str] | None = None,
@@ -1174,10 +1204,11 @@ def build_eval_base_image(
Returns the image tag (e.g. ``skillevaluator-base:a1b2c3d4e5f6``) or ``""``
on failure (callers fall back to full per-task Dockerfiles).
"""
+ evaluator_source = evaluator_skill_path or skill_path
has_custom_dockerfile = (
- skill_path
- and (skill_path / "evals" / "environment" / "Dockerfile").is_file()
- and _validate_custom_dockerfile(skill_path / "evals" / "environment" / "Dockerfile") is None
+ evaluator_source
+ and (evaluator_source / "evals" / "environment" / "Dockerfile").is_file()
+ and _validate_custom_dockerfile(evaluator_source / "evals" / "environment" / "Dockerfile") is None
)
if has_custom_dockerfile:
@@ -1422,48 +1453,101 @@ def _load_evals(evals_path: Path) -> list[dict[str, Any]]:
return normalize_dataset_entries(data)
-def _read_regular_evals_file(path: Path) -> bytes:
- """Read one evaluator file without following links or accepting swaps."""
+def _evaluator_node_fingerprint(metadata: os.stat_result) -> tuple[int, int, int, int, int, int, int]:
+ return (
+ metadata.st_dev,
+ metadata.st_ino,
+ metadata.st_mode,
+ metadata.st_nlink,
+ metadata.st_size,
+ metadata.st_mtime_ns,
+ metadata.st_ctime_ns,
+ )
+
+
+def _snapshot_evaluator_parent_path(path: Path, allowed_root: Path, *, label: str) -> tuple[tuple[Path, tuple], ...]:
+ """Capture every real directory component anchoring an evaluator file."""
+ lexical_path = path.absolute()
+ lexical_root = allowed_root.absolute()
+ try:
+ lexical_path.relative_to(lexical_root)
+ except ValueError as exc:
+ raise ValueError(f"{label} resolves outside its evaluator root: {path}") from exc
+ snapshot: list[tuple[Path, tuple]] = []
+ current = lexical_path.parent
+ while True:
+ try:
+ component = current.lstat()
+ except OSError as exc:
+ raise ValueError(f"Cannot inspect {label.lower()} path: {current}") from exc
+ if _path_is_link_or_reparse(current, component):
+ raise ValueError(f"{label} path must not contain symlinks or reparse points: {current}")
+ if not stat.S_ISDIR(component.st_mode):
+ raise ValueError(f"{label} parent path must contain only directories: {current}")
+ snapshot.append((current, _evaluator_node_fingerprint(component)))
+ if current == lexical_root:
+ break
+ if lexical_root not in current.parents:
+ raise ValueError(f"{label} resolves outside its evaluator root: {path}")
+ current = current.parent
+ return tuple(snapshot)
+
+
+def _validate_evaluator_parent_snapshot(snapshot: tuple[tuple[Path, tuple], ...], *, label: str) -> None:
+ for component_path, expected in snapshot:
+ try:
+ component = component_path.lstat()
+ except OSError as exc:
+ raise ValueError(f"{label} path changed while it was read: {component_path}") from exc
+ if (
+ _path_is_link_or_reparse(component_path, component)
+ or not stat.S_ISDIR(component.st_mode)
+ or _evaluator_node_fingerprint(component) != expected
+ ):
+ raise ValueError(f"{label} path changed while it was read: {component_path}")
+
+
+def _read_regular_evals_file(
+ path: Path,
+ *,
+ label: str = "Eval dataset",
+ max_bytes: int = _MAX_EVALUATOR_FILE_BYTES,
+ allowed_root: Path | None = None,
+) -> bytes:
+ """Read one bounded evaluator file without following links or accepting swaps."""
+ parent_snapshot = (
+ _snapshot_evaluator_parent_path(path, allowed_root, label=label) if allowed_root is not None else ()
+ )
try:
before = path.lstat()
except OSError as exc:
- raise ValueError(f"Cannot inspect eval dataset file: {path}") from exc
+ raise ValueError(f"Cannot inspect {label.lower()} file: {path}") from exc
if _path_is_link_or_reparse(path, before) or not stat.S_ISREG(before.st_mode) or before.st_nlink != 1:
- raise ValueError(f"Eval dataset must be a regular non-linked file: {path}")
- if before.st_size > 64 * 1024 * 1024:
- raise ValueError(f"Eval dataset exceeds the 64 MiB safety limit: {path}")
- flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)
+ raise ValueError(f"{label} must be a regular non-linked file: {path}")
+ if before.st_size > max_bytes:
+ raise ValueError(f"{label} exceeds the {max_bytes}-byte safety limit: {path}")
+ secure_root_path = allowed_root.absolute() if allowed_root is not None else path.parent.absolute()
try:
- descriptor = os.open(path, flags)
- except OSError as exc:
- raise ValueError(f"Cannot safely open eval dataset file: {path}") from exc
+ relative_path = path.absolute().relative_to(secure_root_path)
+ root_metadata = secure_root_path.lstat()
+ with SecureRoot(secure_root_path, expected=root_metadata) as secure_root:
+ payload, opened = secure_root.read_bytes(relative_path, max_bytes, expected=before)
+ except (OSError, SecurePathError, ValueError) as exc:
+ raise ValueError(f"{label} changed while it was read: {path}") from exc
+
try:
- opened = os.fstat(descriptor)
- before_fingerprint = (before.st_dev, before.st_ino, before.st_mode, before.st_nlink, before.st_size)
- opened_fingerprint = (opened.st_dev, opened.st_ino, opened.st_mode, opened.st_nlink, opened.st_size)
- if (
- _path_is_link_or_reparse(path, opened)
- or not stat.S_ISREG(opened.st_mode)
- or opened.st_nlink != 1
- or opened_fingerprint != before_fingerprint
- ):
- raise ValueError(f"Eval dataset changed while it was opened: {path}")
- chunks: list[bytes] = []
- remaining = opened.st_size + 1
- while remaining:
- chunk = os.read(descriptor, min(1024 * 1024, remaining))
- if not chunk:
- break
- chunks.append(chunk)
- remaining -= len(chunk)
- payload = b"".join(chunks)
- after = os.fstat(descriptor)
- after_fingerprint = (after.st_dev, after.st_ino, after.st_mode, after.st_nlink, after.st_size)
- if after_fingerprint != opened_fingerprint or len(payload) != opened.st_size:
- raise ValueError(f"Eval dataset changed while it was read: {path}")
- return payload
- finally:
- os.close(descriptor)
+ named_after = path.lstat()
+ except OSError as exc:
+ raise ValueError(f"{label} changed while it was read: {path}") from exc
+ if (
+ _path_is_link_or_reparse(path, named_after)
+ or _evaluator_node_fingerprint(named_after) != _evaluator_node_fingerprint(opened)
+ or len(payload) != opened.st_size
+ ):
+ raise ValueError(f"{label} changed while it was read: {path}")
+ if parent_snapshot:
+ _validate_evaluator_parent_snapshot(parent_snapshot, label=label)
+ return payload
def _validate_evals_source_directory(skill_path: Path) -> None:
@@ -1519,15 +1603,30 @@ def _write_instruction(task_dir: Path, question: str) -> None:
def _load_mcp_servers(skill_path: Path) -> list[dict[str, Any]]:
"""Load MCP server declarations from evals/environment/mcp_servers.toml."""
- mcp_file = skill_path / "evals" / "environment" / "mcp_servers.toml"
- if not mcp_file.exists():
+ evals_dir = skill_path / "evals"
+ environment_dir = evals_dir / "environment"
+ mcp_file = environment_dir / "mcp_servers.toml"
+ if not os.path.lexists(environment_dir):
+ return []
+ parent_snapshot = _snapshot_evaluator_parent_path(mcp_file, evals_dir, label="MCP server configuration")
+ if not os.path.lexists(mcp_file):
+ _validate_evaluator_parent_snapshot(parent_snapshot, label="MCP server configuration")
return []
try:
import tomllib
except ImportError:
import tomli as tomllib # type: ignore[no-redef]
+ payload = _read_regular_evals_file(
+ mcp_file,
+ label="MCP server configuration",
+ max_bytes=_MAX_MCP_CONFIG_BYTES,
+ allowed_root=evals_dir,
+ )
try:
- data = tomllib.loads(mcp_file.read_text(encoding="utf-8"))
+ data = tomllib.loads(payload.decode("utf-8"))
+ if not isinstance(data, dict):
+ logger.warning("mcp_servers.toml: expected a TOML table, got %s", type(data).__name__)
+ return []
servers = data.get("mcp_servers", [])
if not isinstance(servers, list):
logger.warning("mcp_servers.toml: expected [[mcp_servers]] array, got %s", type(servers).__name__)
@@ -1547,7 +1646,7 @@ def _load_mcp_servers(skill_path: Path) -> list[dict[str, Any]]:
if valid:
logger.debug("Loaded %d MCP server(s) from %s", len(valid), mcp_file)
return valid
- except Exception as e:
+ except (UnicodeDecodeError, tomllib.TOMLDecodeError) as e:
logger.warning("Failed to parse %s: %s", mcp_file, e)
return []
@@ -1739,7 +1838,13 @@ def _has_symlink_component(path: Path, root: Path) -> bool:
return False
-def _copy_custom_grader(task_dir: Path, skill_path: Path, grading_mode: str) -> bool:
+def _copy_custom_grader(
+ task_dir: Path,
+ skill_path: Path,
+ grading_mode: str,
+ *,
+ evals_dir: Path | None = None,
+) -> bool:
"""Copy user custom grader support into a generated task, if configured."""
tests_dir = task_dir / "tests"
tests_dir.mkdir(parents=True, exist_ok=True)
@@ -1748,27 +1853,27 @@ def _copy_custom_grader(task_dir: Path, skill_path: Path, grading_mode: str) ->
if runner_src.exists():
shutil.copy2(runner_src, tests_dir / "custom_grader_runner.py")
+ evals_dir = evals_dir or skill_path / "evals"
grader_candidates = [
- (skill_path / "evals" / "grader.py", tests_dir / "grader.py"),
- (skill_path / "evals" / "grader.sh", tests_dir / "grader.sh"),
- (skill_path / "evals" / "tests" / "grader.py", tests_dir / "grader.py"),
- (skill_path / "evals" / "tests" / "grader.sh", tests_dir / "grader.sh"),
+ (evals_dir / "grader.py", tests_dir / "grader.py"),
+ (evals_dir / "grader.sh", tests_dir / "grader.sh"),
+ (evals_dir / "tests" / "grader.py", tests_dir / "grader.py"),
+ (evals_dir / "tests" / "grader.sh", tests_dir / "grader.sh"),
]
for grader, destination in grader_candidates:
- if _has_symlink_component(grader, skill_path):
+ if _has_symlink_component(grader, evals_dir):
raise ValueError(f"custom grader must be a non-symlinked regular file contained under evals/: {grader}")
if not grader.exists():
continue
if not grader.is_file():
raise ValueError(f"custom grader must be a non-symlinked regular file contained under evals/: {grader}")
- resolved_skill = skill_path.resolve(strict=True)
- resolved_evals = (skill_path / "evals").resolve(strict=True)
+ resolved_evals = evals_dir.resolve(strict=True)
resolved_grader = grader.resolve(strict=True)
- if not resolved_evals.is_relative_to(resolved_skill) or not resolved_grader.is_relative_to(resolved_evals):
+ if not resolved_grader.is_relative_to(resolved_evals):
raise ValueError(f"custom grader must be a non-symlinked regular file contained under evals/: {grader}")
- copy_file_secure(grader, destination, allowed_root=skill_path / "evals")
+ copy_file_secure(grader, destination, allowed_root=evals_dir)
if destination.suffix == ".sh":
destination.chmod(0o755)
return True
@@ -3304,6 +3409,16 @@ def _entry_file_refs(entry: dict[str, Any]) -> list[str]:
return refs
+def _is_evaluator_only_task_input(relative: Path) -> bool:
+ """Return whether an eval-relative path is evaluator control material."""
+ if not relative.parts:
+ return True
+ top_level = relative.parts[0].casefold()
+ return top_level in _EVALUATOR_ONLY_TASK_INPUT_ROOTS or (
+ len(relative.parts) == 1 and top_level in _EVALUATOR_ONLY_TASK_INPUT_FILES
+ )
+
+
def _resolve_entry_file_ref(
ref: str,
*,
@@ -3323,9 +3438,9 @@ def _resolve_entry_file_ref(
raise ValueError(f"evals.json files entry uses unsupported URI scheme: {ref}")
if ref_path.parts and ref_path.parts[0] == "evals":
- candidates = [skill_path / ref_path]
+ candidates = [evals_dir.joinpath(*ref_path.parts[1:])]
else:
- candidates = [evals_dir / ref_path, skill_path / ref_path]
+ candidates = [evals_dir / ref_path]
source_path = next((candidate.absolute() for candidate in candidates if candidate.exists()), None)
if source_path is None:
@@ -3335,6 +3450,9 @@ def _resolve_entry_file_ref(
resolved_evals_dir = evals_dir.resolve()
if source == resolved_evals_dir or not _is_relative_to(source, resolved_evals_dir):
raise ValueError(f"evals.json files entry resolves outside evals/: {ref}")
+ source_relative = source.relative_to(resolved_evals_dir)
+ if _is_evaluator_only_task_input(source_relative):
+ raise ValueError(f"evals.json files entry selects evaluator-only material: {ref}")
if _has_symlink_component(source_path, evals_dir.absolute()):
raise ValueError(f"evals.json files entry must not contain symlinks: {ref}")
@@ -3342,7 +3460,7 @@ def _resolve_entry_file_ref(
if resolved_input_files_dir and _is_relative_to(source, resolved_input_files_dir):
rel = source.relative_to(resolved_input_files_dir)
else:
- rel = source.relative_to(resolved_evals_dir)
+ rel = source_relative
return source_path, rel
@@ -3523,13 +3641,19 @@ def _write_dockerfile(
include_repo = (env_dir / "repo").exists()
include_repo_linked_root = (env_dir / "repo-linked-root").exists()
- custom_env_dir = skill_path / "evals" / "environment" if skill_path else None
+ custom_env_dir = (
+ evals_dir / "environment"
+ if evals_dir is not None
+ else skill_path / "evals" / "environment"
+ if skill_path is not None
+ else None
+ )
if custom_env_dir and custom_env_dir.exists():
private_custom_env = tempfile.TemporaryDirectory(prefix="skillevaluator-custom-environment-")
staged_custom_env = Path(private_custom_env.name) / "environment"
try:
- copytree_secure(custom_env_dir, staged_custom_env, allowed_root=skill_path)
+ copytree_secure(custom_env_dir, staged_custom_env, allowed_root=evals_dir)
if not has_skill and skill_path:
_check_custom_environment_does_not_stage_target(staged_custom_env, skill_path)
@@ -4258,6 +4382,7 @@ def _stage_native_harbor_tasks_into(
skill_path: Path,
output_dir: Path,
*,
+ evaluator_skill_path: Path,
with_skill: bool = True,
reference_skills_dir: Path | None = None,
workspace_skill_paths: list[Path] | None = None,
@@ -4267,6 +4392,7 @@ def _stage_native_harbor_tasks_into(
custom_dockerfile_mode: str = "rebase",
copy_repo: bool = False,
repo_context_exclude_paths: Sequence[Path] = (),
+ private_repo_context_exclude_paths: Sequence[Path] = (),
runtime_env: dict[str, str] | None = None,
verifier_env: dict[str, str] | None = None,
pre_agent_setup: list[str] | None = None,
@@ -4280,7 +4406,8 @@ def _stage_native_harbor_tasks_into(
"""
_validate_runtime_discovery_env(runtime_env)
_validate_runtime_loader_env(runtime_env)
- native_dir = skill_path / "evals" / "harbor"
+ evals_dir = evaluator_skill_path / "evals"
+ native_dir = evals_dir / "harbor"
if not native_dir.exists():
raise FileNotFoundError(f"No native Harbor task source found at {native_dir}")
_validate_staging_output_location(
@@ -4291,7 +4418,7 @@ def _stage_native_harbor_tasks_into(
)
_validate_output_roots_outside_evaluator_sources(skill_path, repo_context_exclude_paths)
_check_native_source_does_not_stage_target(native_dir, skill_path, with_skill=with_skill)
- entries_by_id = _load_entries_by_id(skill_path)
+ entries_by_id = _load_entries_by_id(evaluator_skill_path)
source_task_dirs = _native_task_dirs(native_dir, ignore=_NATIVE_SOURCE_IGNORE)
if not source_task_dirs:
raise ValueError(f"No Harbor task directories with task.toml found in {native_dir}")
@@ -4340,7 +4467,7 @@ def _stage_native_harbor_tasks_into(
if (not custom_grader and grading_mode != "custom_only") or (
not custom_grader and not (tests_dir / "test.sh").exists()
):
- custom_grader = _copy_custom_grader(task_dir, skill_path, grading_mode)
+ custom_grader = _copy_custom_grader(task_dir, skill_path, grading_mode, evals_dir=evals_dir)
if grading_mode in ("default", "default_plus_custom"):
_ensure_skill_evaluator_verifier_env(
@@ -4373,13 +4500,13 @@ def _stage_native_harbor_tasks_into(
f"custom_only native Harbor task '{entry_id}' requires tests/grader.py or tests/test.sh"
)
- if entry is not None and _entry_declares_task_input(entry, skill_path / "evals" / "files"):
+ if entry is not None and _entry_declares_task_input(entry, evals_dir / "files"):
_stage_task_inputs(
task_dir / "environment",
- input_files_dir=skill_path / "evals" / "files",
+ input_files_dir=evals_dir / "files",
entry=entry,
source_skill_path=skill_path,
- evals_dir=skill_path / "evals",
+ evals_dir=evals_dir,
)
_prepare_native_environment(
task_dir,
@@ -4392,7 +4519,7 @@ def _stage_native_harbor_tasks_into(
grading_mode=grading_mode,
repo_context_mode="full" if copy_repo else "linked",
compose_env_names=set(runtime_env or {}),
- repo_context_exclude_paths=repo_context_exclude_paths,
+ repo_context_exclude_paths=(*repo_context_exclude_paths, *private_repo_context_exclude_paths),
agent_workdir=native_agent_workdir,
)
@@ -4402,6 +4529,32 @@ def _stage_native_harbor_tasks_into(
return task_dirs
+def _private_task_staging_parent(
+ skill_path: Path,
+ *,
+ reference_skills_dir: Path | None,
+ workspace_skill_paths: Sequence[Path],
+) -> Path:
+ """Choose a temp parent outside every runtime skill source.
+
+ Preserve the normal system temporary location unless TMPDIR places task
+ staging inside a runtime skill source. In that case, use a sibling of the
+ target skill and ascend only when that location is itself inside a
+ configured reference or workspace source.
+ """
+ external_sources = [*workspace_skill_paths]
+ if reference_skills_dir is not None:
+ external_sources.append(reference_skills_dir)
+ candidate = Path(tempfile.gettempdir())
+ if not any(_path_is_excluded(candidate, (source,)) for source in (skill_path, *external_sources)):
+ return candidate
+
+ candidate = skill_path.parent
+ while candidate.parent != candidate and any(_path_is_excluded(candidate, (source,)) for source in external_sources):
+ candidate = candidate.parent
+ return candidate
+
+
def stage_native_harbor_tasks(
skill_path: Path,
output_dir: Path,
@@ -4420,9 +4573,32 @@ def stage_native_harbor_tasks(
pre_agent_setup: list[str] | None = None,
task_resources: dict[str, int] | None = None,
agent_workdir: str | None = None,
+ evaluator_skill_path: Path | None = None,
) -> list[Path]:
"""Stage native tasks privately, then publish one exact output snapshot."""
+ if evaluator_skill_path is None:
+ with private_evaluator_skill_snapshot(skill_path, task_source="native_harbor") as private_skill_path:
+ return stage_native_harbor_tasks(
+ skill_path,
+ output_dir,
+ with_skill=with_skill,
+ reference_skills_dir=reference_skills_dir,
+ workspace_skill_paths=workspace_skill_paths,
+ workspace_mode=workspace_mode,
+ grading_mode=grading_mode,
+ base_image=base_image,
+ custom_dockerfile_mode=custom_dockerfile_mode,
+ copy_repo=copy_repo,
+ repo_context_exclude_paths=repo_context_exclude_paths,
+ runtime_env=runtime_env,
+ verifier_env=verifier_env,
+ pre_agent_setup=pre_agent_setup,
+ task_resources=task_resources,
+ agent_workdir=agent_workdir,
+ evaluator_skill_path=private_skill_path,
+ )
+
validate_output_provenance_key_location(
skill_path,
output_dir,
@@ -4440,12 +4616,21 @@ def stage_native_harbor_tasks(
if output_requires_provenance:
validate_generated_output_replacement(output_dir)
- with tempfile.TemporaryDirectory(prefix="skillevaluator-native-tasks-") as temporary:
+ private_staging_parent = _private_task_staging_parent(
+ skill_path,
+ reference_skills_dir=reference_skills_dir,
+ workspace_skill_paths=workspace_skill_paths or (),
+ )
+ with tempfile.TemporaryDirectory(
+ prefix="skillevaluator-native-tasks-",
+ dir=private_staging_parent,
+ ) as temporary:
private_root = Path(temporary).resolve(strict=True)
private_output = private_root / "dataset"
private_tasks = _stage_native_harbor_tasks_into(
skill_path,
private_output,
+ evaluator_skill_path=evaluator_skill_path,
with_skill=with_skill,
reference_skills_dir=reference_skills_dir,
workspace_skill_paths=workspace_skill_paths,
@@ -4454,7 +4639,12 @@ def stage_native_harbor_tasks(
base_image=base_image,
custom_dockerfile_mode=custom_dockerfile_mode,
copy_repo=copy_repo,
- repo_context_exclude_paths=(*repo_context_exclude_paths, output_dir, private_root),
+ repo_context_exclude_paths=(
+ *repo_context_exclude_paths,
+ output_dir,
+ private_root,
+ ),
+ private_repo_context_exclude_paths=(evaluator_skill_path.parent,),
runtime_env=runtime_env,
verifier_env=verifier_env,
pre_agent_setup=pre_agent_setup,
@@ -4501,6 +4691,7 @@ def _generate_harbor_tasks_into(
skill_path: Path,
output_dir: Path,
*,
+ evaluator_skill_path: Path,
with_skill: bool = True,
reference_skills_dir: Path | None = None,
workspace_skill_paths: list[Path] | None = None,
@@ -4510,6 +4701,7 @@ def _generate_harbor_tasks_into(
custom_dockerfile_mode: str = "rebase",
copy_repo: bool = False,
repo_context_exclude_paths: Sequence[Path] = (),
+ private_repo_context_exclude_paths: Sequence[Path] = (),
runtime_env: dict[str, str] | None = None,
verifier_env: dict[str, str] | None = None,
pre_agent_setup: list[str] | None = None,
@@ -4550,8 +4742,8 @@ def _generate_harbor_tasks_into(
_validate_runtime_discovery_env(runtime_env)
_validate_runtime_loader_env(runtime_env)
agent_workdir = _validated_agent_workdir(agent_workdir)
- evals_dir = skill_path / "evals"
- evals_file = find_evals_file(skill_path)
+ evals_dir = evaluator_skill_path / "evals"
+ evals_file = find_evals_file(evaluator_skill_path)
_validate_staging_output_location(
skill_path,
output_dir,
@@ -4573,7 +4765,7 @@ def _generate_harbor_tasks_into(
if not input_files_dir.exists():
input_files_dir = None
- mcp_servers = _load_mcp_servers(skill_path)
+ mcp_servers = _load_mcp_servers(evaluator_skill_path)
prepared_entries = _preflight_generated_tasks(entries, output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
task_dirs: list[str] = []
@@ -4604,7 +4796,7 @@ def _generate_harbor_tasks_into(
agent_workdir=agent_workdir,
)
_copy_verifier(task_dir)
- custom_grader = _copy_custom_grader(task_dir, skill_path, grading_mode)
+ custom_grader = _copy_custom_grader(task_dir, skill_path, grading_mode, evals_dir=evals_dir)
_write_entry_json(
task_dir,
normalized_entry,
@@ -4631,7 +4823,7 @@ def _generate_harbor_tasks_into(
repo_context_skill_path=skill_path,
repo_context_mode="full" if copy_repo else "linked",
compose_env_names=set(runtime_env or {}),
- repo_context_exclude_paths=repo_context_exclude_paths,
+ repo_context_exclude_paths=(*repo_context_exclude_paths, *private_repo_context_exclude_paths),
agent_workdir=agent_workdir,
)
@@ -4651,6 +4843,361 @@ def _generate_harbor_tasks_into(
return task_paths
+@dataclass(frozen=True)
+class _EvaluatorProjectionSelection:
+ """Evaluator paths selected before the single secure-copy transaction."""
+
+ exact_paths: frozenset[tuple[str, ...]]
+ whole_roots: frozenset[tuple[str, ...]]
+ signature: tuple[Any, ...]
+
+
+def _projection_path_exists(path: Path) -> bool:
+ """Check lexical presence without silently dropping a broken link."""
+ return os.path.lexists(path)
+
+
+def _selected_environment_projection(evals_dir: Path) -> tuple[set[tuple[str, ...]], set[tuple[str, ...]]]:
+ """Select the custom environment as one compatibility-preserving unit."""
+ exact: set[tuple[str, ...]] = set()
+ whole: set[tuple[str, ...]] = set()
+ environment = evals_dir / "environment"
+ if not _projection_path_exists(environment):
+ return exact, whole
+ # Custom Docker handling first creates a secure copy of this complete tree
+ # before it validates Dockerfile/Compose and selects sidecars. Keeping the
+ # directory whole preserves authored environment behavior while unrelated
+ # evals/ siblings remain outside the snapshot.
+ whole.add(("environment",))
+ return exact, whole
+
+
+def _selected_grader_paths(evals_dir: Path) -> tuple[tuple[str, ...], ...]:
+ candidates = (("grader.py",), ("grader.sh",), ("tests", "grader.py"), ("tests", "grader.sh"))
+ return tuple(relative for relative in candidates if _projection_path_exists(evals_dir.joinpath(*relative)))
+
+
+def _resolved_projection_task_source(
+ requested_task_source: str | None,
+ evals_config: dict[str, Any],
+ *,
+ dataset_path: Path | None,
+ native_dir: Path,
+) -> str:
+ configured = requested_task_source
+ if configured is None:
+ harbor_config = evals_config.get("harbor", {})
+ configured = harbor_config.get("task_source", "auto") if isinstance(harbor_config, dict) else "auto"
+ if configured not in {"auto", "evals_json", "native_harbor"}:
+ configured = "auto"
+ if configured == "auto":
+ if dataset_path is not None:
+ return "evals_json"
+ if native_dir.exists():
+ return "native_harbor"
+ return configured
+
+
+def _native_projection_entry_ids(native_dir: Path) -> tuple[str, ...]:
+ """Read native task IDs without following unsafe authored nodes.
+
+ The secure projection copy validates the complete selected ``harbor/`` tree
+ as one transaction. Selection only needs immediate task identities so it
+ can avoid fixtures referenced exclusively by generated entries.
+ """
+ try:
+ native_metadata = native_dir.lstat()
+ parent_metadata = native_dir.parent.lstat()
+ except OSError:
+ return ()
+ if (
+ _path_is_link_or_reparse(native_dir, native_metadata)
+ or not stat.S_ISDIR(native_metadata.st_mode)
+ or not stat.S_ISDIR(parent_metadata.st_mode)
+ or native_metadata.st_dev != parent_metadata.st_dev
+ ):
+ return ()
+ try:
+ children = sorted(native_dir.iterdir())
+ except OSError:
+ return ()
+ ignored = set(_NATIVE_SOURCE_IGNORE(os.fspath(native_dir), [path.name for path in children]))
+ entry_ids: list[str] = []
+ for task_dir in children:
+ if task_dir.name in ignored or task_dir.name.startswith((".", "_")):
+ continue
+ try:
+ task_metadata = task_dir.lstat()
+ except OSError:
+ continue
+ if (
+ _path_is_link_or_reparse(task_dir, task_metadata)
+ or not stat.S_ISDIR(task_metadata.st_mode)
+ or task_metadata.st_dev != native_metadata.st_dev
+ ):
+ continue
+ task_toml = task_dir / "task.toml"
+ if not _projection_path_exists(task_toml):
+ continue
+ entry_id = task_dir.name
+ try:
+ payload = _read_regular_evals_file(
+ task_toml,
+ label="Native Harbor task configuration",
+ allowed_root=native_dir,
+ )
+ data = tomllib.loads(payload.decode("utf-8"))
+ metadata = data.get("metadata", {}) if isinstance(data, dict) else {}
+ if isinstance(metadata, dict) and metadata.get("entry_id"):
+ entry_id = str(metadata["entry_id"])
+ except (UnicodeError, ValueError):
+ # Staging owns the detailed invalid-task diagnostic after the secure
+ # snapshot exists. Directory-name fallback remains non-invasive.
+ pass
+ entry_ids.append(entry_id)
+ return tuple(sorted(entry_ids))
+
+
+def _decode_projection_config(payload: bytes) -> tuple[dict[str, Any], bool]:
+ """Decode only the source-selection hint from already-secured config bytes."""
+ import yaml
+
+ try:
+ raw = yaml.safe_load(payload.decode("utf-8"))
+ except (UnicodeError, yaml.YAMLError):
+ return {}, False
+ if not isinstance(raw, dict):
+ return {}, False
+ harbor = raw.get("harbor")
+ if harbor is not None and not isinstance(harbor, dict):
+ return {}, False
+ return raw, True
+
+
+def _evaluator_projection_selection(
+ skill_path: Path,
+ *,
+ task_source: str | None,
+ bind_full_evidence_sources: bool,
+) -> _EvaluatorProjectionSelection:
+ """Resolve the evaluator projection and a post-copy selection signature."""
+ import yaml
+
+ from skillevaluator.tier3.dataset_utils import normalize_dataset_entries
+ from skillevaluator.tier3.evals_config import CONFIG_FILENAMES
+
+ evals_dir = skill_path / "evals"
+ config_path = next(
+ (evals_dir / name for name in CONFIG_FILENAMES if _projection_path_exists(evals_dir / name)),
+ None,
+ )
+ evals_config: dict[str, Any] = {}
+ config_valid = True
+ config_digest: str | None = None
+ if config_path is not None:
+ config_payload = _read_regular_evals_file(
+ config_path,
+ label="Eval configuration",
+ max_bytes=_MAX_EVALS_CONFIG_BYTES,
+ allowed_root=evals_dir,
+ )
+ config_digest = hashlib.sha256(config_payload).hexdigest()
+ evals_config, config_valid = _decode_projection_config(config_payload)
+
+ dataset_path = find_evals_file(skill_path)
+ dataset_valid = True
+ entries: list[dict[str, Any]] = []
+ dataset_digest: str | None = None
+ if dataset_path is not None:
+ dataset_payload = _read_regular_evals_file(dataset_path, allowed_root=evals_dir)
+ dataset_digest = hashlib.sha256(dataset_payload).hexdigest()
+ try:
+ decoded = dataset_payload.decode("utf-8")
+ suffix = dataset_path.suffix.casefold()
+ if suffix == ".jsonl":
+ raw_entries: Any = [json.loads(line) for raw_line in decoded.splitlines() if (line := raw_line.strip())]
+ elif suffix in {".yaml", ".yml"}:
+ raw_entries = yaml.safe_load(decoded)
+ elif suffix == ".json":
+ raw_entries = json.loads(decoded)
+ else:
+ raise ValueError(f"Unsupported dataset format: {suffix}")
+ entries = normalize_dataset_entries(raw_entries)
+ except (UnicodeError, TypeError, ValueError, yaml.YAMLError):
+ # Invalid datasets do not have a meaningful fixture selection. Copy
+ # the dataset itself and let normal task staging report its error.
+ dataset_valid = False
+
+ resolved_task_source = _resolved_projection_task_source(
+ task_source,
+ evals_config,
+ dataset_path=dataset_path,
+ native_dir=evals_dir / "harbor",
+ )
+ native_entry_ids: tuple[str, ...] = ()
+ native_dir = evals_dir / "harbor"
+ if resolved_task_source == "native_harbor" and _projection_path_exists(native_dir):
+ native_entry_ids = _native_projection_entry_ids(native_dir)
+ exact_paths: set[tuple[str, ...]] = set()
+ whole_roots: set[tuple[str, ...]] = set()
+ if config_path is not None:
+ exact_paths.add((config_path.name,))
+ if dataset_path is not None:
+ exact_paths.add((dataset_path.name,))
+ if resolved_task_source == "native_harbor" and _projection_path_exists(evals_dir / "harbor"):
+ whole_roots.add(("harbor",))
+
+ environment_exact, environment_whole = _selected_environment_projection(evals_dir)
+ exact_paths.update(environment_exact)
+ whole_roots.update(environment_whole)
+
+ available_grader_paths = _selected_grader_paths(evals_dir)
+ grader_paths = (
+ available_grader_paths
+ if bind_full_evidence_sources and resolved_task_source == "evals_json"
+ else available_grader_paths[:1]
+ )
+ exact_paths.update(grader_paths)
+
+ fixture_signature: tuple[Any, ...] = ("invalid-or-absent-dataset",)
+ if dataset_path is not None and dataset_valid:
+ files_dir = evals_dir / "files"
+ native_entry_id_set = set(native_entry_ids)
+ fixture_entries = (
+ [entry for entry in entries if str(entry.get("id")) in native_entry_id_set]
+ if resolved_task_source == "native_harbor"
+ else entries
+ )
+ implicit_shared_files = (
+ (bind_full_evidence_sources and resolved_task_source == "evals_json")
+ or any("files" not in entry for entry in fixture_entries)
+ ) and _projection_path_exists(files_dir)
+ selected_refs: set[tuple[str, ...]] = set()
+ if implicit_shared_files:
+ whole_roots.add(("files",))
+ for entry in fixture_entries:
+ for ref in _entry_file_refs(entry):
+ source, _staged_relative = _resolve_entry_file_ref(
+ ref,
+ skill_path=skill_path,
+ evals_dir=evals_dir,
+ input_files_dir=files_dir if files_dir.exists() else None,
+ )
+ relative = source.absolute().relative_to(evals_dir.absolute()).parts
+ try:
+ source_metadata = source.lstat()
+ except OSError:
+ exact_paths.add(relative)
+ selected_refs.add(relative)
+ continue
+ if stat.S_ISDIR(source_metadata.st_mode) and not _path_is_link_or_reparse(source, source_metadata):
+ whole_roots.add(relative)
+ else:
+ exact_paths.add(relative)
+ selected_refs.add(relative)
+ fixture_signature = (
+ "all-files" if implicit_shared_files else "declared-only",
+ tuple(sorted(selected_refs)),
+ )
+
+ signature = (
+ config_path.name if config_path is not None else None,
+ config_valid,
+ config_digest,
+ resolved_task_source,
+ dataset_path.name if dataset_path is not None else None,
+ dataset_valid,
+ dataset_digest,
+ bind_full_evidence_sources,
+ native_entry_ids,
+ fixture_signature,
+ grader_paths,
+ tuple(sorted(environment_exact)),
+ tuple(sorted(environment_whole)),
+ )
+ return _EvaluatorProjectionSelection(
+ exact_paths=frozenset(exact_paths),
+ whole_roots=frozenset(whole_roots),
+ signature=signature,
+ )
+
+
+def _projection_includes(
+ relative: tuple[str, ...],
+ *,
+ exact_paths: frozenset[tuple[str, ...]],
+ whole_roots: frozenset[tuple[str, ...]],
+) -> bool:
+ if any(relative == exact[: len(relative)] for exact in exact_paths):
+ return True
+ return any(relative == whole[: len(relative)] or whole == relative[: len(whole)] for whole in whole_roots)
+
+
+@contextmanager
+def private_evaluator_skill_snapshot(
+ skill_path: Path,
+ *,
+ task_source: str | None = None,
+ bind_full_evidence_sources: bool = False,
+) -> Iterator[Path]:
+ """Yield one selective, immutable evaluator projection for a complete run."""
+ source_evals_dir = skill_path / "evals"
+ _validate_evals_source_directory(skill_path)
+ if not source_evals_dir.is_dir():
+ raise FileNotFoundError(f"No evaluator source directory found at {source_evals_dir}")
+
+ selection = _evaluator_projection_selection(
+ skill_path,
+ task_source=task_source,
+ bind_full_evidence_sources=bind_full_evidence_sources,
+ )
+ source_evals_canonical = source_evals_dir.resolve(strict=True)
+
+ with tempfile.TemporaryDirectory(prefix="skillevaluator-evals-snapshot-") as snapshot_root_text:
+ snapshot_root = Path(snapshot_root_text)
+ evaluator_skill_path = snapshot_root / skill_path.name
+ evaluator_skill_path.mkdir()
+ try:
+ snapshot_source_relative = snapshot_root.resolve(strict=True).relative_to(source_evals_canonical).parts
+ except (OSError, ValueError):
+ snapshot_source_relative = ()
+
+ def _ignore_unselected_evaluator_paths(current: str, names: list[str]) -> set[str]:
+ try:
+ current_relative = Path(current).resolve(strict=True).relative_to(source_evals_canonical).parts
+ except (OSError, ValueError):
+ return set(names)
+ ignored = {
+ name
+ for name in names
+ if not _projection_includes(
+ (*current_relative, name),
+ exact_paths=selection.exact_paths,
+ whole_roots=selection.whole_roots,
+ )
+ }
+ if snapshot_source_relative:
+ ignored.update(name for name in names if (*current_relative, name) == snapshot_source_relative)
+ if current_relative and current_relative[0].casefold() == "harbor":
+ ignored.update(_NATIVE_SOURCE_IGNORE(current, names))
+ return ignored
+
+ copytree_secure(
+ source_evals_dir,
+ evaluator_skill_path / "evals",
+ ignore=_ignore_unselected_evaluator_paths,
+ allowed_root=source_evals_dir,
+ )
+ copied_selection = _evaluator_projection_selection(
+ evaluator_skill_path,
+ task_source=task_source,
+ bind_full_evidence_sources=bind_full_evidence_sources,
+ )
+ if copied_selection.signature != selection.signature:
+ raise ValueError("Evaluator source selection changed while its private snapshot was created")
+ yield evaluator_skill_path
+
+
def generate_harbor_tasks(
skill_path: Path,
output_dir: Path,
@@ -4669,8 +5216,35 @@ def generate_harbor_tasks(
pre_agent_setup: list[str] | None = None,
task_resources: dict[str, int] | None = None,
agent_workdir: str | None = None,
+ evaluator_skill_path: Path | None = None,
) -> list[Path]:
- """Generate tasks privately, then publish one exact output snapshot."""
+ """Generate tasks from one private evals snapshot, then publish exactly."""
+
+ if evaluator_skill_path is None:
+ if find_evals_file(skill_path) is None:
+ raise FileNotFoundError(f"No evals dataset found in {skill_path / 'evals'}")
+ with private_evaluator_skill_snapshot(skill_path, task_source="evals_json") as private_skill_path:
+ return generate_harbor_tasks(
+ skill_path,
+ output_dir,
+ with_skill=with_skill,
+ reference_skills_dir=reference_skills_dir,
+ workspace_skill_paths=workspace_skill_paths,
+ workspace_mode=workspace_mode,
+ grading_mode=grading_mode,
+ base_image=base_image,
+ custom_dockerfile_mode=custom_dockerfile_mode,
+ copy_repo=copy_repo,
+ repo_context_exclude_paths=repo_context_exclude_paths,
+ runtime_env=runtime_env,
+ verifier_env=verifier_env,
+ pre_agent_setup=pre_agent_setup,
+ task_resources=task_resources,
+ agent_workdir=agent_workdir,
+ evaluator_skill_path=private_skill_path,
+ )
+ if find_evals_file(evaluator_skill_path) is None:
+ raise FileNotFoundError(f"No evals dataset found in {evaluator_skill_path / 'evals'}")
validate_output_provenance_key_location(
skill_path,
@@ -4689,12 +5263,21 @@ def generate_harbor_tasks(
if output_requires_provenance:
validate_generated_output_replacement(output_dir)
- with tempfile.TemporaryDirectory(prefix="skillevaluator-generated-tasks-") as temporary:
+ private_staging_parent = _private_task_staging_parent(
+ skill_path,
+ reference_skills_dir=reference_skills_dir,
+ workspace_skill_paths=workspace_skill_paths or (),
+ )
+ with tempfile.TemporaryDirectory(
+ prefix="skillevaluator-generated-tasks-",
+ dir=private_staging_parent,
+ ) as temporary:
private_root = Path(temporary).resolve(strict=True)
private_output = private_root / "dataset"
private_tasks = _generate_harbor_tasks_into(
skill_path,
private_output,
+ evaluator_skill_path=evaluator_skill_path,
with_skill=with_skill,
reference_skills_dir=reference_skills_dir,
workspace_skill_paths=workspace_skill_paths,
@@ -4703,7 +5286,12 @@ def generate_harbor_tasks(
base_image=base_image,
custom_dockerfile_mode=custom_dockerfile_mode,
copy_repo=copy_repo,
- repo_context_exclude_paths=(*repo_context_exclude_paths, output_dir, private_root),
+ repo_context_exclude_paths=(
+ *repo_context_exclude_paths,
+ output_dir,
+ private_root,
+ ),
+ private_repo_context_exclude_paths=(evaluator_skill_path.parent,),
runtime_env=runtime_env,
verifier_env=verifier_env,
pre_agent_setup=pre_agent_setup,
diff --git a/src/skillevaluator/tier3/harbor/report_data.py b/src/skillevaluator/tier3/harbor/report_data.py
index 0f7c10e6..96348efc 100644
--- a/src/skillevaluator/tier3/harbor/report_data.py
+++ b/src/skillevaluator/tier3/harbor/report_data.py
@@ -338,7 +338,11 @@ def _condition_status(agent_info: dict[str, Any], condition: str) -> str:
return str(data.get("execution_status") or "unknown") if isinstance(data, dict) else "unknown"
-def load_agent_data(results_dir: Path) -> dict[str, dict[str, Any]]:
+def load_agent_data(
+ results_dir: Path,
+ *,
+ allow_legacy_missing_status: bool = False,
+) -> dict[str, dict[str, Any]]:
"""Load per-agent summaries, rewards, lift, and execution coverage."""
agents: dict[str, dict[str, Any]] = {}
selection_diagnostics: list[dict[str, Any]] = []
@@ -391,6 +395,8 @@ def load_agent_data(results_dir: Path) -> dict[str, dict[str, Any]]:
if "pass_at_k" in data:
agent_info[pass_key] = data["pass_at_k"]
status = data.get("execution_status")
+ if status is None and allow_legacy_missing_status:
+ status = "succeeded"
if status not in {"succeeded", "failed", "skipped"}:
status = "unknown"
errors = data.get("execution_errors")
diff --git a/src/skillevaluator/tier3/harbor/runner.py b/src/skillevaluator/tier3/harbor/runner.py
index e208bdad..e4c58d0d 100644
--- a/src/skillevaluator/tier3/harbor/runner.py
+++ b/src/skillevaluator/tier3/harbor/runner.py
@@ -18,7 +18,7 @@ import time
import tomllib
from collections.abc import Iterator, Mapping
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, as_completed, wait
-from contextlib import contextmanager, suppress
+from contextlib import ExitStack, contextmanager, suppress
from dataclasses import dataclass
from datetime import UTC, datetime
from functools import wraps
@@ -35,6 +35,7 @@ from skillevaluator.tier3.harbor.adapter import (
build_eval_base_image,
find_evals_file,
generate_harbor_tasks,
+ private_evaluator_skill_snapshot,
stage_native_harbor_tasks,
validate_output_provenance_key_location,
validate_results_root_location,
@@ -1658,9 +1659,12 @@ def _run_harbor_eval_impl(
override_memory_mb: int | None = None,
override_storage_mb: int | None = None,
progress_reporter: ProgressReporter | None = None,
+ _evaluator_skill_path: Path | None = None,
+ _monotonic_start: float | None = None,
) -> dict[str, Any]:
"""Run a public Harbor evaluation with and without the target skill."""
- started_at = time.monotonic()
+ forwarded = dict(locals()) if _evaluator_skill_path is None else None
+ started_at = _monotonic_start if _monotonic_start is not None else time.monotonic()
reporter = safe_progress_reporter(progress_reporter or NullProgressReporter())
if env_mode not in HARBOR_ENV_MODES:
reporter.emit(ProgressEvent(stage="configuration", state="failed", detail="unsupported environment"))
@@ -1677,9 +1681,25 @@ def _run_harbor_eval_impl(
reporter.emit(ProgressEvent(stage="configuration", state="failed", detail=str(exc)))
return {"error": [str(exc)]}
+ if _evaluator_skill_path is None:
+ assert forwarded is not None
+ forwarded.pop("skill_path")
+ forwarded.pop("agents")
+ with ExitStack() as snapshot_stack:
+ try:
+ evaluator_skill_path = snapshot_stack.enter_context(private_evaluator_skill_snapshot(skill_path))
+ except (FileNotFoundError, ValueError) as exc:
+ reporter.emit(ProgressEvent(stage="configuration", state="failed", detail=str(exc)))
+ return {"error": [str(exc)]}
+ forwarded["_evaluator_skill_path"] = evaluator_skill_path
+ forwarded["_monotonic_start"] = started_at
+ return _run_harbor_eval_impl(skill_path, agents, **forwarded)
+
+ evaluator_skill_path = _evaluator_skill_path
+
try:
provider = resolve_llm_provider()
- config, config_path = load_evals_config(skill_path)
+ config, config_path = load_evals_config(evaluator_skill_path)
except (ProviderConfigurationError, EvalsConfigError) as exc:
reporter.emit(ProgressEvent(stage="configuration", state="failed", detail=str(exc)))
return {"error": [str(exc)]}
@@ -1812,8 +1832,8 @@ def _run_harbor_eval_impl(
reporter.emit(ProgressEvent(stage="with-skill-tasks", state="failed", detail=str(exc)))
return {"error": [str(exc)]}
- evals_exists = find_evals_file(skill_path) is not None
- native_exists = (skill_path / "evals" / "harbor").exists()
+ evals_exists = find_evals_file(evaluator_skill_path) is not None
+ native_exists = (evaluator_skill_path / "evals" / "harbor").exists()
if task_source == "auto":
task_source = "evals_json" if evals_exists else "native_harbor" if native_exists else ""
if task_source == "evals_json" and not evals_exists:
@@ -1884,6 +1904,7 @@ def _run_harbor_eval_impl(
skill_path.resolve(),
reference_skills_dir,
workspace_skill_paths=workspace_skills,
+ evaluator_skill_path=evaluator_skill_path,
excluded_roots=(root,),
force_rebuild=base_image_mode == "rebuild",
)
@@ -1930,6 +1951,7 @@ def _run_harbor_eval_impl(
pre_agent_setup=harbor_config.get("pre_agent_setup", []),
task_resources=resource_config,
agent_workdir=harbor_config.get("agent_workdir"),
+ evaluator_skill_path=evaluator_skill_path,
)
task_names = [task.name for task in task_paths]
if expected_task_names is None:
@@ -1960,6 +1982,7 @@ def _run_harbor_eval_impl(
pre_agent_setup=harbor_config.get("pre_agent_setup", []),
task_resources=resource_config,
agent_workdir=harbor_config.get("agent_workdir"),
+ evaluator_skill_path=evaluator_skill_path,
)
if not skip_baseline:
reporter.emit(ProgressEvent(stage="baseline-tasks", state="ready", detail="baseline inputs staged"))
@@ -2166,7 +2189,7 @@ def _run_harbor_eval_impl(
raise
reporter.emit(ProgressEvent(stage="collection", state="complete", detail="Harbor results collected"))
run_config = {
- "config_file": str(config_path.relative_to(skill_path)) if config_path else "none",
+ "config_file": str(config_path.relative_to(evaluator_skill_path)) if config_path else "none",
"harbor": {
"environment": {"value": env_mode, "source": env_mode_source},
"n_attempts": n_attempts,
diff --git a/src/skillevaluator/tier3/harbor/secure_copy.py b/src/skillevaluator/tier3/harbor/secure_copy.py
index 2042318a..fe89d4e8 100644
--- a/src/skillevaluator/tier3/harbor/secure_copy.py
+++ b/src/skillevaluator/tier3/harbor/secure_copy.py
@@ -38,6 +38,7 @@ _CHUNK_SIZE = 1024 * 1024
_REPARSE_POINT = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
_DIRECTORY_FLAGS = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
_FILE_FLAGS = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_NOCTTY", 0)
+_WINDOWS_CHMOD_SEMANTICS = os.name == "nt"
_DESCRIPTOR_BACKEND = (
os.name == "posix"
and hasattr(os, "O_NOFOLLOW")
@@ -1548,6 +1549,45 @@ def _safe_remove_fallback(path: Path) -> None:
raise UnsafeStagingError(f"refusing to remove unsafe fallback staging entry: {path}")
+def _fallback_mode_matches(actual: int, expected: int) -> bool:
+ """Check the mode bits that the platform's path-based chmod can set."""
+
+ if _WINDOWS_CHMOD_SEMANTICS:
+ return bool(actual & stat.S_IWRITE) == bool(expected & stat.S_IWRITE)
+ return stat.S_IMODE(actual) & 0o777 == expected
+
+
+def _apply_fallback_open_file_mode(path: Path, descriptor: int, mode: int) -> None:
+ """Apply a mode through the strongest API available and verify identity."""
+
+ opened_before = os.fstat(descriptor)
+ named_before = path.lstat()
+ if (
+ _is_link(named_before)
+ or not stat.S_ISREG(opened_before.st_mode)
+ or not stat.S_ISREG(named_before.st_mode)
+ or opened_before.st_nlink != 1
+ or named_before.st_nlink != 1
+ or _node_identity(named_before) != _node_identity(opened_before)
+ ):
+ raise UnsafeStagingError("fallback staging file changed before its mode was applied")
+ fchmod = getattr(os, "fchmod", None)
+ if fchmod is not None:
+ fchmod(descriptor, mode)
+ else:
+ path.chmod(mode)
+ opened_after = os.fstat(descriptor)
+ named_after = path.lstat()
+ if (
+ _is_link(named_after)
+ or _node_identity(opened_after) != _node_identity(opened_before)
+ or _node_identity(named_after) != _node_identity(opened_after)
+ or not _fallback_mode_matches(opened_after.st_mode, mode)
+ or not _fallback_mode_matches(named_after.st_mode, mode)
+ ):
+ raise UnsafeStagingError("fallback staging file identity or mode changed while applying its mode")
+
+
def _copy_manifest_file_fallback(manifest: _TreeManifest, entry: _ManifestEntry, destination: Path) -> None:
source = manifest.source.joinpath(*entry.parts)
before = source.lstat()
@@ -1567,7 +1607,7 @@ def _copy_manifest_file_fallback(manifest: _TreeManifest, entry: _ManifestEntry,
destination_descriptor = -1
try:
destination_descriptor = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
- os.fchmod(destination_descriptor, 0o600)
+ _apply_fallback_open_file_mode(destination, destination_descriptor, 0o600)
digest = hashlib.sha256()
while data := os.read(source_descriptor, _CHUNK_SIZE):
digest.update(data)
@@ -1613,13 +1653,13 @@ def _apply_modes_fallback(stage: Path, manifests: tuple[_TreeManifest, ...]) ->
for entry in (item for item in entries.values() if item.kind == "file"):
path = stage.joinpath(*entry.parts)
path.chmod(entry.mode)
- if stat.S_IMODE(path.lstat().st_mode) & 0o777 != entry.mode:
+ if not _fallback_mode_matches(path.lstat().st_mode, entry.mode):
raise UnsafeStagingError("published fallback file mode could not be applied")
directories = [item for item in entries.values() if item.kind == "directory"]
for entry in sorted(directories, key=lambda item: len(item.parts), reverse=True):
path = stage.joinpath(*entry.parts)
path.chmod(entry.mode)
- if stat.S_IMODE(path.lstat().st_mode) & 0o777 != entry.mode:
+ if not _fallback_mode_matches(path.lstat().st_mode, entry.mode):
raise UnsafeStagingError("published fallback directory mode could not be applied")
@@ -1645,7 +1685,7 @@ def _validate_fallback_tree_exact_pass(
_is_link(before)
or not stat.S_ISDIR(before.st_mode)
or before.st_dev != root_device
- or stat.S_IMODE(before.st_mode) & 0o777 != expected_mode
+ or not _fallback_mode_matches(before.st_mode, expected_mode)
):
raise UnsafeStagingError("fallback staging directory is unsafe")
before_fingerprint = _fingerprint(before)
@@ -1667,7 +1707,7 @@ def _validate_fallback_tree_exact_pass(
not stat.S_ISREG(metadata.st_mode)
or metadata.st_nlink != 1
or metadata.st_size != child_entry.size
- or stat.S_IMODE(metadata.st_mode) & 0o777 != expected_file_mode
+ or not _fallback_mode_matches(metadata.st_mode, expected_file_mode)
):
raise UnsafeStagingError("fallback staging file is unsafe")
if (
@@ -1741,7 +1781,7 @@ def _expose_fallback_tree_root_exact(stage: Path, manifests: tuple[_TreeManifest
root_mode = manifests[-1].root.mode
stage.chmod(root_mode)
after = stage.lstat()
- if _node_identity(after) != _node_identity(before) or stat.S_IMODE(after.st_mode) & 0o777 != root_mode:
+ if _node_identity(after) != _node_identity(before) or not _fallback_mode_matches(after.st_mode, root_mode):
raise UnsafeStagingError("published fallback root identity or mode changed during exposure")
@@ -1773,7 +1813,7 @@ def _prepare_fallback_moved_backup(
private_mode = 0o700 if kind == "directory" else 0o600
backup.chmod(private_mode)
private = backup.lstat()
- if _node_identity(private) != _node_identity(expected) or stat.S_IMODE(private.st_mode) & 0o777 != private_mode:
+ if _node_identity(private) != _node_identity(expected) or not _fallback_mode_matches(private.st_mode, private_mode):
raise UnsafeStagingError("fallback destination backup could not be made private")
@@ -2076,7 +2116,7 @@ def _validate_fallback_file_exact(
or not stat.S_ISREG(metadata.st_mode)
or metadata.st_nlink != 1
or metadata.st_size != manifest.entry.size
- or stat.S_IMODE(metadata.st_mode) & 0o777 != expected_mode
+ or not _fallback_mode_matches(metadata.st_mode, expected_mode)
):
raise UnsafeStagingError("fallback staging file has an unsafe type, link count, size, or mode")
if (
@@ -2098,7 +2138,7 @@ def _stage_fallback_file(manifest: _FileManifest, stage: Path) -> None:
destination_descriptor = -1
try:
destination_descriptor = os.open(stage, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
- os.fchmod(destination_descriptor, 0o600)
+ _apply_fallback_open_file_mode(stage, destination_descriptor, 0o600)
digest = hashlib.sha256()
while data := os.read(source_descriptor, _CHUNK_SIZE):
digest.update(data)
diff --git a/src/skillevaluator/tier3/results_location.py b/src/skillevaluator/tier3/results_location.py
index 9d62af99..0a57feb3 100644
--- a/src/skillevaluator/tier3/results_location.py
+++ b/src/skillevaluator/tier3/results_location.py
@@ -6,6 +6,7 @@
from __future__ import annotations
import json
+import math
import os
import stat
import subprocess
@@ -14,9 +15,21 @@ from datetime import datetime
from pathlib import Path
from uuid import uuid4
+from skillevaluator.tier3.output_provenance import GENERATED_OUTPUT_MARKER
+from skillevaluator.utils.secure_fs import SecurePathError, SecureRoot
+
ENV_RESULTS_DIR = "SKILLEVALUATOR_RESULTS_DIR"
+
_RUN_TIMESTAMP_FORMATS = (("%Y%m%d_%H%M%S", 15), ("%Y-%m-%d_%H%M%S", 17))
+# A current completed run carries ``run_config.json`` plus an atomically
+# written ``result.json`` whose run identity matches its directory. Historical
+# runs created before run-level results instead require a usable per-agent
+# summary and must predate the generated-output provenance marker. When a
+# historical summary carries status or attempt coverage, those must be complete.
+_RUN_COMPLETION_ARTIFACTS = ("run_config.json",)
+_FINAL_RESULT_ARTIFACT = "result.json"
+
def _expand(path: str | Path) -> Path:
return Path(path).expanduser().resolve()
@@ -94,13 +107,14 @@ def iter_candidate_results_roots(
def _run_timestamp(name: str) -> datetime | None:
+ """Parse the timestamp prefix shared by current and legacy run names."""
for timestamp_format, prefix_length in _RUN_TIMESTAMP_FORMATS:
prefix = name[:prefix_length]
suffix = name[prefix_length:]
if suffix and not suffix.startswith("_"):
continue
try:
- return datetime.strptime(prefix, timestamp_format) # noqa: DTZ007 -- directory names have no timezone
+ return datetime.strptime(prefix, timestamp_format) # noqa: DTZ007 -- run names have no timezone
except ValueError:
continue
return None
@@ -120,16 +134,453 @@ def _path_is_link_or_reparse(path: Path, metadata: os.stat_result) -> bool:
return True
+def _node_fingerprint(metadata: os.stat_result) -> tuple[int, int, int, int, int, int, int]:
+ """Return the fields that must remain stable while reading an artifact."""
+ return (
+ metadata.st_dev,
+ metadata.st_ino,
+ metadata.st_mode,
+ metadata.st_nlink,
+ metadata.st_size,
+ metadata.st_mtime_ns,
+ metadata.st_ctime_ns,
+ )
+
+
+def _is_single_link_regular(path: Path, metadata: os.stat_result) -> bool:
+ return not _path_is_link_or_reparse(path, metadata) and stat.S_ISREG(metadata.st_mode) and metadata.st_nlink == 1
+
+
+def _read_stable_json(
+ secure_root: SecureRoot,
+ path: Path,
+ before: os.stat_result,
+) -> tuple[object, os.stat_result, bytes] | None:
+ """Read one JSON artifact through the pinned run root."""
+ if not _is_single_link_regular(path, before):
+ return None
+ try:
+ relative_path = path.absolute().relative_to(secure_root.root)
+ raw, opened = secure_root.read_bytes(relative_path, before.st_size, expected=before)
+ after = path.lstat()
+ payload = json.loads(raw.decode("utf-8"))
+ except (UnicodeDecodeError, json.JSONDecodeError, OSError, SecurePathError, ValueError):
+ return None
+ if (
+ not _is_single_link_regular(path, after)
+ or _node_fingerprint(opened) != _node_fingerprint(before)
+ or _node_fingerprint(after) != _node_fingerprint(opened)
+ or len(raw) != opened.st_size
+ ):
+ return None
+ return payload, opened, raw
+
+
+def _artifact_snapshot_is_unchanged(
+ secure_root: SecureRoot,
+ snapshot: tuple[Path, bytes, os.stat_result],
+) -> bool:
+ """Re-read one artifact to detect ordinary changes during validation."""
+ path, expected_raw, expected_metadata = snapshot
+ try:
+ current_metadata = path.lstat()
+ except OSError:
+ return False
+ observed = _read_stable_json(secure_root, path, current_metadata)
+ return bool(
+ observed is not None
+ and observed[2] == expected_raw
+ and _node_fingerprint(observed[1]) == _node_fingerprint(expected_metadata)
+ )
+
+
+def _legacy_run_agents(payload: object) -> tuple[str, ...] | None:
+ """Return configured result-agent keys from an authentic historical config."""
+ if not isinstance(payload, dict) or not isinstance(payload.get("harbor"), dict) or not payload["harbor"]:
+ return None
+ if payload.get("task_source") not in {"evals_json", "native_harbor"}:
+ return None
+ agents = payload.get("agents")
+ if not isinstance(agents, dict) or not agents:
+ return None
+ normalized: list[str] = []
+ for agent, metadata in agents.items():
+ if (
+ not isinstance(agent, str)
+ or not agent
+ or agent in {".", ".."}
+ or "/" in agent
+ or "\\" in agent
+ or not isinstance(metadata, dict)
+ or not isinstance(metadata.get("agent"), str)
+ or not metadata["agent"].strip()
+ or not isinstance(metadata.get("model"), str)
+ or not metadata["model"].strip()
+ or not isinstance(metadata.get("source"), str)
+ or not metadata["source"].strip()
+ or not isinstance(metadata.get("occurrence"), str)
+ or not metadata["occurrence"].isdigit()
+ or int(metadata["occurrence"]) < 1
+ ):
+ return None
+ normalized.append(agent)
+ return tuple(sorted(normalized))
+
+
+def _is_finite_number(value: object) -> bool:
+ if isinstance(value, bool) or not isinstance(value, int | float):
+ return False
+ try:
+ return math.isfinite(value)
+ except OverflowError:
+ return False
+
+
+def _is_nonnegative_int(value: object) -> bool:
+ return isinstance(value, int) and not isinstance(value, bool) and value >= 0
+
+
+def _legacy_pass_at_k_is_complete(
+ payload: object,
+ *,
+ num_trials: int,
+ require_scored_attempt: bool,
+ expected_scored_attempts: int | None = None,
+ allow_coverage_failure: bool = False,
+) -> bool:
+ """Validate the historical collector's complete pass-at-k structure."""
+ if not isinstance(payload, dict):
+ return False
+ k = payload.get("k")
+ threshold = payload.get("pass_threshold")
+ stop_on_pass = payload.get("stop_on_pass")
+ passed_cases = payload.get("passed_cases")
+ failed_cases = payload.get("failed_cases")
+ attempts_used = payload.get("attempts_used")
+ total_cases = payload.get("total_cases")
+ max_attempts = payload.get("max_attempts_possible")
+ rate = payload.get("rate")
+ average_attempts = payload.get("avg_attempts_used")
+ if (
+ not isinstance(k, int)
+ or isinstance(k, bool)
+ or k < 1
+ or not _is_finite_number(threshold)
+ or not 0 <= threshold <= 1
+ or not isinstance(stop_on_pass, bool)
+ or any(not _is_nonnegative_int(value) for value in (passed_cases, failed_cases, attempts_used, max_attempts))
+ or not isinstance(total_cases, int)
+ or isinstance(total_cases, bool)
+ or total_cases < 1
+ or passed_cases + failed_cases != total_cases
+ or max_attempts != total_cases * k
+ or (not allow_coverage_failure and attempts_used > max_attempts)
+ or not _is_finite_number(rate)
+ or not 0 <= rate <= 1
+ or not math.isclose(rate, round(passed_cases / total_cases, 4), abs_tol=1e-9)
+ or not _is_finite_number(average_attempts)
+ or average_attempts < 0
+ or not math.isclose(average_attempts, round(attempts_used / total_cases, 4), abs_tol=1e-9)
+ ):
+ return False
+ extra_cases = payload.get("extra_cases")
+ if (
+ not isinstance(extra_cases, list)
+ or any(not isinstance(case, str) or not case for case in extra_cases)
+ or len(extra_cases) != len(set(extra_cases))
+ ):
+ return False
+ extra_case_names = set(extra_cases)
+ cases = payload.get("cases")
+ if not isinstance(cases, dict) or not cases:
+ return False
+ total_attempt_rows = 0
+ expected_attempt_rows = 0
+ observed_passed_cases = 0
+ observed_expected_cases = 0
+ for case_name, case in cases.items():
+ if not isinstance(case_name, str) or not isinstance(case, dict):
+ return False
+ case_attempts_used = case.get("attempts_used")
+ attempts_skipped = case.get("attempts_skipped")
+ attempts_missing = case.get("attempts_missing")
+ attempts = case.get("attempts")
+ if (
+ not isinstance(case.get("passed"), bool)
+ or not _is_nonnegative_int(case_attempts_used)
+ or not _is_nonnegative_int(attempts_skipped)
+ or not _is_nonnegative_int(attempts_missing)
+ or not isinstance(attempts, list)
+ or len(attempts) != case_attempts_used
+ ):
+ return False
+ first_pass_attempt = case.get("first_pass_attempt")
+ if first_pass_attempt is not None and (
+ not isinstance(first_pass_attempt, int)
+ or isinstance(first_pass_attempt, bool)
+ or first_pass_attempt < 1
+ or first_pass_attempt > len(attempts)
+ ):
+ return False
+ for ordinal, attempt in enumerate(attempts, start=1):
+ if (
+ not isinstance(attempt, dict)
+ or attempt.get("attempt") != ordinal
+ or not isinstance(attempt.get("trial"), str)
+ or not _is_finite_number(attempt.get("score"))
+ or not isinstance(attempt.get("passed"), bool)
+ ):
+ return False
+ score = attempt.get("score")
+ if attempt["passed"] != (score >= threshold):
+ return False
+ passing_ordinals = [ordinal for ordinal, attempt in enumerate(attempts, start=1) if attempt["passed"]]
+ expected_first_pass = passing_ordinals[0] if passing_ordinals else None
+ attempt_scores = [attempt["score"] for attempt in attempts]
+ expected_best_score = max(attempt_scores) if attempt_scores else None
+ best_score = case.get("best_score")
+ if (
+ case["passed"] != bool(passing_ordinals)
+ or first_pass_attempt != expected_first_pass
+ or (
+ (expected_best_score is None and best_score is not None)
+ or (
+ expected_best_score is not None
+ and (
+ not _is_finite_number(best_score)
+ or not math.isclose(best_score, expected_best_score, abs_tol=1e-9)
+ )
+ )
+ )
+ ):
+ return False
+ total_attempt_rows += len(attempts)
+ if case_name not in extra_case_names:
+ observed_expected_cases += 1
+ observed_passed_cases += int(case["passed"])
+ expected_attempt_rows += len(attempts)
+ if not allow_coverage_failure and case_attempts_used + attempts_skipped + attempts_missing != k:
+ return False
+ elif case.get("extra_case") is not True:
+ return False
+ return (
+ extra_case_names.issubset(cases)
+ and observed_expected_cases == total_cases
+ and observed_passed_cases == passed_cases
+ and expected_attempt_rows == attempts_used
+ and total_attempt_rows <= num_trials
+ and (expected_scored_attempts is None or total_attempt_rows == expected_scored_attempts)
+ and (not require_scored_attempt or total_attempt_rows > 0)
+ )
+
+
+def _legacy_dimensions_are_valid(payload: object) -> bool:
+ if not isinstance(payload, dict):
+ return False
+ for name, dimension in payload.items():
+ if (
+ not isinstance(name, str)
+ or not isinstance(dimension, dict)
+ or not _is_finite_number(dimension.get("score"))
+ ):
+ return False
+ sources = dimension.get("sources")
+ if not isinstance(sources, dict) or any(
+ not isinstance(source, str) or not _is_finite_number(weight) for source, weight in sources.items()
+ ):
+ return False
+ return True
+
+
+def _legacy_summary_is_complete(payload: object, *, expected_agent: str) -> bool:
+ """Return whether a pre-result summary satisfies an authentic era schema."""
+ if not isinstance(payload, dict):
+ return False
+ if payload.get("agent") != expected_agent:
+ return False
+ if (
+ not isinstance(payload.get("model"), str)
+ or not payload["model"].strip()
+ or not isinstance(payload.get("model_source"), str)
+ or not payload["model_source"].strip()
+ ):
+ return False
+ metric_set = payload.get("metric_set")
+ metrics = payload.get("metrics")
+ if not isinstance(metric_set, str) or not metric_set or not isinstance(metrics, list):
+ return False
+ if any(not isinstance(metric, str) or not metric for metric in metrics) or not _legacy_dimensions_are_valid(
+ payload.get("dimensions")
+ ):
+ return False
+
+ num_trials = payload.get("num_trials")
+ if not _is_nonnegative_int(num_trials):
+ return False
+
+ completion_fields = {
+ "execution_status",
+ "execution_errors",
+ "expected_attempts",
+ "scored_attempts",
+ "job_failure",
+ "trial_failures",
+ }
+ present_completion_fields = completion_fields.intersection(payload)
+ scored_attempts: int | None = None
+ if present_completion_fields and present_completion_fields != completion_fields:
+ return False
+ if present_completion_fields:
+ execution_status = payload.get("execution_status")
+ execution_errors = payload.get("execution_errors")
+ job_failure = payload.get("job_failure")
+ trial_failures = payload.get("trial_failures")
+ if (
+ execution_status not in {"succeeded", "failed"}
+ or not isinstance(execution_errors, list)
+ or any(not isinstance(error, str) or not error for error in execution_errors)
+ or not isinstance(job_failure, str)
+ or not isinstance(trial_failures, list)
+ or any(not isinstance(failure, dict) for failure in trial_failures)
+ ):
+ return False
+ expected_attempts = payload.get("expected_attempts")
+ scored_attempts = payload.get("scored_attempts")
+ if not _is_nonnegative_int(expected_attempts) or not _is_nonnegative_int(scored_attempts):
+ return False
+ if execution_status == "succeeded" and (
+ execution_errors
+ or job_failure
+ or trial_failures
+ or expected_attempts < 1
+ or scored_attempts != expected_attempts
+ ):
+ return False
+ if execution_status == "failed" and not execution_errors:
+ return False
+ elif num_trials < 1:
+ return False
+
+ scores = payload.get("scores")
+ custom_scores = payload.get("custom_scores")
+ if not isinstance(scores, dict) or not isinstance(custom_scores, dict):
+ return False
+ for score_group in (scores, custom_scores):
+ for value in score_group.values():
+ if isinstance(value, bool) or not isinstance(value, int | float):
+ return False
+ try:
+ if not math.isfinite(value):
+ return False
+ except OverflowError:
+ return False
+ if scores and any(metric not in metrics for metric in scores):
+ return False
+ status = payload.get("execution_status")
+ require_scored_attempt = status != "failed"
+ if not _legacy_pass_at_k_is_complete(
+ payload.get("pass_at_k"),
+ num_trials=num_trials,
+ require_scored_attempt=require_scored_attempt,
+ expected_scored_attempts=scored_attempts,
+ allow_coverage_failure=status == "failed",
+ ):
+ return False
+ if status == "failed":
+ return True
+ return bool(scores or custom_scores) or (metric_set == "custom-only" and metrics == [])
+
+
+def _legacy_completion_mtime(
+ candidate: Path,
+ secure_root: SecureRoot,
+ snapshots: list[tuple[Path, bytes, os.stat_result]],
+ expected_agents: tuple[str, ...],
+) -> int | None:
+ """Return completion time for an unmarked pre-result run, if safely readable."""
+ marker = candidate / GENERATED_OUTPUT_MARKER
+ if os.path.lexists(marker):
+ # Current runs are marked before any evaluation artifact is written.
+ # Any marker presence, including a corrupt marker, prevents an
+ # incomplete current run from downgrading into the legacy contract.
+ return None
+
+ completion_mtimes: list[int] = []
+ try:
+ children = sorted(candidate.iterdir(), key=lambda path: path.name)
+ agent_dirs: dict[str, Path] = {}
+ for child in children:
+ if child.name.startswith("_"):
+ continue
+ child_metadata = child.lstat()
+ if _path_is_link_or_reparse(child, child_metadata):
+ return None
+ if not stat.S_ISDIR(child_metadata.st_mode):
+ continue
+ if child.name not in expected_agents:
+ return None
+ agent_dirs[child.name] = child
+
+ if set(agent_dirs) != set(expected_agents):
+ return None
+
+ for agent_name in expected_agents:
+ agent_dir = agent_dirs[agent_name]
+
+ condition_dir = agent_dir / "with-skill"
+ if os.path.lexists(condition_dir):
+ condition_metadata = condition_dir.lstat()
+ if _path_is_link_or_reparse(condition_dir, condition_metadata):
+ return None
+ if not stat.S_ISDIR(condition_metadata.st_mode):
+ return None
+ nested_summary = condition_dir / "summary.json"
+ flat_summary = agent_dir / "summary.json"
+ summary_paths = [path for path in (nested_summary, flat_summary) if os.path.lexists(path)]
+ if len(summary_paths) != 1:
+ return None
+
+ baseline_dir = agent_dir / "without-skill"
+ if os.path.lexists(baseline_dir):
+ baseline_metadata = baseline_dir.lstat()
+ if _path_is_link_or_reparse(baseline_dir, baseline_metadata) or not stat.S_ISDIR(
+ baseline_metadata.st_mode
+ ):
+ return None
+ baseline_summary = baseline_dir / "summary.json"
+ if not os.path.lexists(baseline_summary):
+ return None
+ summary_paths.append(baseline_summary)
+
+ for summary_path in summary_paths:
+ summary_metadata = summary_path.lstat()
+ if not _is_single_link_regular(summary_path, summary_metadata):
+ return None
+ observed = _read_stable_json(secure_root, summary_path, summary_metadata)
+ if observed is None or not _legacy_summary_is_complete(observed[0], expected_agent=agent_name):
+ return None
+ snapshots.append((summary_path, observed[2], observed[1]))
+ completion_mtimes.append(observed[1].st_mtime_ns)
+ except OSError:
+ return None
+
+ if os.path.lexists(marker):
+ return None
+ return max(completion_mtimes, default=None)
+
+
def run_directory_sort_key(
candidate: Path,
*,
require_completed_result: bool = False,
) -> tuple[datetime, int, str] | None:
- """Return shared newest-first ordering metadata for a result directory.
+ """Return shared ordering metadata for one result directory.
- Timestamp-shaped directories are current runs and are visible only after
- their final ``result.json`` identifies the directory's run id. Historical
- non-timestamp summary directories remain available to compare workflows.
+ Timestamped runs are accepted when they carry either a valid final result
+ matching the directory's run ID or the stricter pre-result legacy contract,
+ and are ordered by parsed timestamp, completion mtime, then name. Compare
+ also keeps accepting summary-only non-timestamp directories, placing them
+ behind timestamped runs.
"""
if candidate.name.startswith((".", "_")) or candidate.name == "latest":
return None
@@ -141,37 +592,57 @@ def run_directory_sort_key(
return None
timestamp = _run_timestamp(candidate.name)
- is_current_run = timestamp is not None
- if not is_current_run:
+ if timestamp is None:
if require_completed_result:
return None
- timestamp = datetime.min # noqa: DTZ901 -- result directory timestamps are intentionally timezone-free
+ return datetime.min, -1, candidate.name # noqa: DTZ901 -- legacy names have no timezone
- completed = False
- completion_mtime = -1
try:
- result_path = candidate / "result.json"
- before = result_path.lstat()
- if _path_is_link_or_reparse(result_path, before) or not stat.S_ISREG(before.st_mode) or before.st_nlink != 1:
- return None
- result = json.loads(result_path.read_text(encoding="utf-8"))
- after = result_path.lstat()
- if (
- _path_is_link_or_reparse(result_path, after)
- or not stat.S_ISREG(after.st_mode)
- or after.st_nlink != 1
- or (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns)
- != (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns)
- ):
+ snapshots: list[tuple[Path, bytes, os.stat_result]] = []
+ with SecureRoot(candidate, expected=candidate_metadata) as secure_root:
+ run_config_path = candidate / _RUN_COMPLETION_ARTIFACTS[0]
+ run_config_metadata = run_config_path.lstat()
+ observed_run_config = _read_stable_json(secure_root, run_config_path, run_config_metadata)
+ if observed_run_config is None or not isinstance(observed_run_config[0], dict):
+ return None
+ snapshots.append((run_config_path, observed_run_config[2], observed_run_config[1]))
+
+ result_path = candidate / _FINAL_RESULT_ARTIFACT
+ try:
+ result_metadata = result_path.lstat()
+ except FileNotFoundError:
+ expected_agents = _legacy_run_agents(observed_run_config[0])
+ if expected_agents is None:
+ return None
+ legacy_completion_mtime = _legacy_completion_mtime(
+ candidate,
+ secure_root,
+ snapshots,
+ expected_agents,
+ )
+ if legacy_completion_mtime is None:
+ return None
+ completion_mtime = max(legacy_completion_mtime, observed_run_config[1].st_mtime_ns)
+ else:
+ observed_result = _read_stable_json(secure_root, result_path, result_metadata)
+ if observed_result is None:
+ return None
+ result = observed_result[0]
+ if not isinstance(result, dict) or result.get("run_id") != candidate.name:
+ return None
+ snapshots.append((result_path, observed_result[2], observed_result[1]))
+ completion_mtime = observed_result[1].st_mtime_ns
+
+ if not all(_artifact_snapshot_is_unchanged(secure_root, snapshot) for snapshot in snapshots):
+ return None
+
+ candidate_after = candidate.lstat()
+ if _path_is_link_or_reparse(candidate, candidate_after) or _node_fingerprint(
+ candidate_after
+ ) != _node_fingerprint(candidate_metadata):
return None
- completion_mtime = after.st_mtime_ns
- completed = isinstance(result, dict) and result.get("run_id") == candidate.name
- except (FileNotFoundError, json.JSONDecodeError, OSError):
- pass
- if (is_current_run or require_completed_result) and not completed:
+ except (OSError, SecurePathError):
return None
- if not completed:
- completion_mtime = -1
return timestamp, completion_mtime, candidate.name
@@ -194,8 +665,35 @@ def ordered_run_directories(root: Path, *, completed_only: bool = False) -> list
return [candidate for _, candidate in ordered]
-def _newest_completed_run(root: Path) -> Path | None:
- """Return the newest complete timestamped run without relying on symlinks."""
+def _is_completed_run_dir(path: Path) -> bool:
+ """Return whether ``path`` is a *completed* timestamped run directory.
+
+ A candidate must match the run-name timestamp pattern, carry its run
+ configuration, and satisfy either the current final-result contract or the
+ pre-result legacy-summary contract. This keeps the symlink-less fallback in
+ :func:`resolve_latest_results` from resolving aborted/empty runs,
+ ``_harbor-*`` staging dirs, ``astra-cleanup``, or the ``<plugin>-plugin-eval``
+ wrapper directory (MR !29 review 59316310).
+ """
+ return run_directory_sort_key(path, require_completed_result=True) is not None
+
+
+def is_legacy_completed_run_dir(path: Path) -> bool:
+ """Return whether ``path`` satisfies the authenticated pre-result contract."""
+ if _run_timestamp(path.name) is None:
+ return False
+ if os.path.lexists(path / _FINAL_RESULT_ARTIFACT) or os.path.lexists(path / GENERATED_OUTPUT_MARKER):
+ return False
+ return run_directory_sort_key(path, require_completed_result=True) is not None
+
+
+def _newest_run_dir(root: Path) -> Path | None:
+ """Return the newest *completed* timestamped run directory under ``root``.
+
+ This is the cross-platform fallback for when the ``latest`` symlink was
+ never created. Only completed runs are considered, with same-second runs
+ ordered by their completion artifact mtime and then deterministic name.
+ """
return next(iter(ordered_run_directories(root, completed_only=True)), None)
@@ -204,11 +702,12 @@ def publish_latest_results(root: Path, run_id: str) -> bool:
temporary = root / f".latest-{os.getpid()}-{uuid4().hex}.tmp"
try:
temporary.symlink_to(run_id)
- os.replace(temporary, root / "latest") # noqa: PTH105 -- explicit atomic replacement is the contract
+ os.replace(temporary, root / "latest") # noqa: PTH105 -- atomic replacement is the contract
except OSError:
+ return False
+ finally:
with suppress(OSError):
temporary.unlink()
- return False
return True
@@ -218,42 +717,52 @@ def resolve_latest_results(
*,
environ: dict[str, str] | None = None,
) -> Path:
- """Return the best available ``latest`` results path for read workflows."""
+ """Return the best available ``latest`` results path for read workflows.
+
+ Prefers each root's valid ``latest`` symlink. The link must resolve to an
+ immediate, completed child of that root. When it is absent or invalid
+ (including native Windows where symlink creation is not permitted), falls
+ back to the newest completed timestamped run directory.
+ """
roots = iter_candidate_results_roots(
skill_path,
cli_results_dir,
environ=environ,
)
- first_missing_latest: Path | None = None
+ first_absent_latest: Path | None = None
for root in roots:
- latest = root / "latest"
- if not os.path.lexists(latest) and first_missing_latest is None:
- first_missing_latest = latest
- if latest.is_symlink():
+ candidate = root / "latest"
+ if not os.path.lexists(candidate) and first_absent_latest is None:
+ first_absent_latest = candidate
+ if candidate.is_symlink():
try:
- link_target = latest.readlink()
- if link_target.is_absolute() or len(link_target.parts) != 1 or link_target.parts[0] in {"", ".", ".."}:
- raise ValueError("latest must name one relative run directory")
+ link_target = candidate.readlink()
resolved_root = root.resolve(strict=True)
- target = latest.resolve(strict=True)
- except (OSError, RuntimeError, ValueError):
+ target = candidate.resolve(strict=True)
+ except (OSError, RuntimeError):
pass
else:
if (
- link_target.name == target.name
+ not link_target.is_absolute()
+ and len(link_target.parts) == 1
+ and link_target.name == target.name
and target.parent == resolved_root
- and run_directory_sort_key(target, require_completed_result=True) is not None
+ and _is_completed_run_dir(target)
):
- return latest
- fallback = _newest_completed_run(root)
- if fallback is not None:
- return fallback
- if first_missing_latest is not None:
- return first_missing_latest
+ return candidate
+ newest = _newest_run_dir(root)
+ if newest is not None:
+ return newest
+ if first_absent_latest is not None:
+ return first_absent_latest
+ # Never hand an invalid existing ``latest`` entry back to callers: most
+ # immediately test ``exists()`` or resolve symlinks and would otherwise
+ # consume the very directory/link rejected above. A unique non-existent
+ # path preserves the historical Path-returning API and its absence check.
while True:
- unavailable = roots[0] / f".latest-unavailable-{uuid4().hex}"
- if not os.path.lexists(unavailable):
- return unavailable
+ unresolved = roots[0] / f".latest-unavailable-{uuid4().hex}"
+ if not os.path.lexists(unresolved):
+ return unresolved
def resolve_explicit_or_latest_results(
diff --git a/src/skillevaluator/utils/secure_fs.py b/src/skillevaluator/utils/secure_fs.py
new file mode 100644
index 00000000..870039a1
--- /dev/null
+++ b/src/skillevaluator/utils/secure_fs.py
@@ -0,0 +1,729 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Descriptor-anchored, bounded reads for evaluator-owned artifacts.
+
+Selected files are opened relative to a pinned root without following links.
+The reader verifies identity, type, link count, size, and timestamps before and
+after the read. Native Windows file handles allow reads but deny concurrent
+writes and deletes until the selected-file descriptor closes.
+"""
+
+from __future__ import annotations
+
+import errno
+import os
+import stat
+from dataclasses import dataclass
+from pathlib import Path
+
+_OPEN_SUPPORTS_DIR_FD = os.open in os.supports_dir_fd
+
+_WINDOWS_FILE_READ_DATA = 0x1
+_WINDOWS_FILE_TRAVERSE = 0x20
+_WINDOWS_FILE_READ_ATTRIBUTES = 0x80
+_WINDOWS_SYNCHRONIZE = 0x100000
+_WINDOWS_SHARE_READ = 0x1
+_WINDOWS_SHARE_READ_WRITE = _WINDOWS_SHARE_READ | 0x2
+_WINDOWS_FILE_OPEN = 1
+_WINDOWS_FILE_DIRECTORY_FILE = 0x1
+_WINDOWS_FILE_SYNCHRONOUS_IO_NONALERT = 0x20
+_WINDOWS_FILE_NON_DIRECTORY_FILE = 0x40
+_WINDOWS_FILE_OPEN_FOR_BACKUP_INTENT = 0x00004000
+_WINDOWS_FILE_OPEN_REPARSE_POINT = 0x00200000
+_WINDOWS_OBJ_CASE_INSENSITIVE = 0x40
+_WINDOWS_OBJ_DONT_REPARSE = 0x1000
+_WINDOWS_OBJECT_ATTRIBUTES_FLAGS = _WINDOWS_OBJ_CASE_INSENSITIVE | _WINDOWS_OBJ_DONT_REPARSE
+_WINDOWS_DIRECTORY_READ_ACCESS = _WINDOWS_FILE_READ_ATTRIBUTES | _WINDOWS_FILE_TRAVERSE | _WINDOWS_SYNCHRONIZE
+_WINDOWS_FILE_READ_ACCESS = _WINDOWS_FILE_READ_DATA | _WINDOWS_FILE_READ_ATTRIBUTES | _WINDOWS_SYNCHRONIZE
+_WINDOWS_DIRECTORY_OPEN_OPTIONS = (
+ _WINDOWS_FILE_DIRECTORY_FILE
+ | _WINDOWS_FILE_SYNCHRONOUS_IO_NONALERT
+ | _WINDOWS_FILE_OPEN_FOR_BACKUP_INTENT
+ | _WINDOWS_FILE_OPEN_REPARSE_POINT
+)
+_WINDOWS_FILE_OPEN_OPTIONS = (
+ _WINDOWS_FILE_NON_DIRECTORY_FILE | _WINDOWS_FILE_SYNCHRONOUS_IO_NONALERT | _WINDOWS_FILE_OPEN_REPARSE_POINT
+)
+
+
+class SecurePathError(ValueError):
+ """An unsafe, racy, inaccessible, or unbounded filesystem input."""
+
+ def __init__(
+ self,
+ code: str,
+ message: str,
+ *,
+ relative_path: str = ".",
+ metadata: dict[str, int] | None = None,
+ ) -> None:
+ super().__init__(message)
+ self.code = code
+ self.relative_path = relative_path
+ self.metadata = metadata or {}
+
+
+@dataclass(frozen=True)
+class _WindowsHandleMetadata:
+ """Stable metadata queried from one open native Windows handle."""
+
+ attributes: int
+ volume_serial: int
+ file_id: int
+ size: int
+ link_count: int
+ last_write_time: int = 0
+
+
+def stat_is_link_or_reparse(metadata: os.stat_result) -> bool:
+ """Return whether metadata identifies a symlink or Windows reparse point."""
+ reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
+ return stat.S_ISLNK(metadata.st_mode) or bool(getattr(metadata, "st_file_attributes", 0) & reparse_flag)
+
+
+def _absolute_no_resolve(path: Path) -> Path:
+ """Return an absolute lexical path without resolving links."""
+ return Path(os.path.abspath(os.fspath(path))) # noqa: PTH100
+
+
+def _relative_path(path: Path) -> Path:
+ if path.is_absolute() or not path.parts or any(part in {"", ".", ".."} for part in path.parts):
+ raise SecurePathError("unsafe_path", f"Path must be relative and normalized: {path.as_posix()}")
+ return path
+
+
+def _raise_unsafe_file(relative: Path, *, hardlink: bool = False) -> None:
+ if hardlink:
+ raise SecurePathError(
+ "unsafe_hardlink",
+ f"Refusing hard-linked selected file with link count greater than one: {relative.as_posix()}",
+ relative_path=relative.as_posix(),
+ )
+ raise SecurePathError(
+ "unsafe_path",
+ f"Refusing selected path that is not a regular file: {relative.as_posix()}",
+ relative_path=relative.as_posix(),
+ )
+
+
+def _validate_opened_file(
+ metadata: os.stat_result,
+ relative_path: Path,
+ expected: os.stat_result | None,
+) -> None:
+ if stat_is_link_or_reparse(metadata):
+ raise SecurePathError(
+ "unsafe_path",
+ f"Refusing selected symlink or reparse point: {relative_path.as_posix()}",
+ relative_path=relative_path.as_posix(),
+ )
+ if not stat.S_ISREG(metadata.st_mode):
+ _raise_unsafe_file(relative_path)
+ if getattr(metadata, "st_nlink", 1) != 1:
+ _raise_unsafe_file(relative_path, hardlink=True)
+ if expected is not None:
+ changed = not os.path.samestat(metadata, expected)
+ for attribute in ("st_size", "st_mtime_ns", "st_ctime_ns"):
+ if getattr(metadata, attribute, None) != getattr(expected, attribute, None):
+ changed = True
+ if changed:
+ raise SecurePathError(
+ "unsafe_path",
+ f"Selected file changed identity or contents while being opened: {relative_path.as_posix()}",
+ relative_path=relative_path.as_posix(),
+ )
+
+
+def _validate_directory_snapshot(
+ metadata: os.stat_result,
+ relative_path: Path,
+ expected: os.stat_result,
+) -> None:
+ """Require one directory identity and entry snapshot to stay stable."""
+ changed = (
+ stat_is_link_or_reparse(metadata)
+ or not stat.S_ISDIR(metadata.st_mode)
+ or not os.path.samestat(metadata, expected)
+ )
+ for attribute in ("st_size", "st_mtime_ns", "st_ctime_ns"):
+ if getattr(metadata, attribute, None) != getattr(expected, attribute, None):
+ changed = True
+ if changed:
+ label = relative_path.as_posix()
+ raise SecurePathError(
+ "unsafe_path",
+ f"Secure root directory changed identity or snapshot: {label}",
+ relative_path=label,
+ )
+
+
+def _open_absolute_directory_posix(path: Path) -> int:
+ flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0)
+ try:
+ expected = path.lstat()
+ except OSError as exc:
+ raise SecurePathError("unsafe_root", f"Cannot inspect declared root safely: {exc}") from exc
+ if stat_is_link_or_reparse(expected) or not stat.S_ISDIR(expected.st_mode):
+ raise SecurePathError("unsafe_root", f"Declared root is a symlink or non-directory: {path}")
+ try:
+ descriptor = os.open(path, flags)
+ except OSError as exc:
+ raise SecurePathError("unsafe_root", f"Cannot securely open declared root: {exc}") from exc
+ try:
+ opened = os.fstat(descriptor)
+ if not stat.S_ISDIR(opened.st_mode) or not os.path.samestat(expected, opened):
+ raise SecurePathError("unsafe_root", "Declared root changed while being opened.")
+ return descriptor
+ except BaseException:
+ os.close(descriptor)
+ raise
+
+
+class SecureRoot:
+ """Descriptor-anchored reads beneath one verified regular root."""
+
+ def __init__(self, root: Path, *, expected: os.stat_result | None = None) -> None:
+ self.root = _absolute_no_resolve(root)
+ self._expected = expected
+ self._root_fd: int | None = None
+ self._windows_root_handles: list[int] = []
+ self._entered = False
+
+ def __enter__(self) -> SecureRoot:
+ if self._entered:
+ raise SecurePathError("unsafe_root", "Secure root context is already active.")
+ try:
+ metadata = self.root.lstat()
+ except OSError as exc:
+ raise SecurePathError("invalid_root", f"Cannot inspect secure root: {exc}") from exc
+ if stat_is_link_or_reparse(metadata):
+ raise SecurePathError("unsafe_root", f"Secure root is a symlink or reparse point: {self.root.name}")
+ if not stat.S_ISDIR(metadata.st_mode):
+ raise SecurePathError("invalid_root", f"Secure root is not a regular directory: {self.root}")
+ if self._expected is not None:
+ _validate_directory_snapshot(metadata, Path(), self._expected)
+
+ if os.name == "posix":
+ if not (hasattr(os, "O_DIRECTORY") and hasattr(os, "O_NOFOLLOW") and _OPEN_SUPPORTS_DIR_FD):
+ raise SecurePathError(
+ "secure_open_unavailable",
+ "This platform cannot guarantee descriptor-anchored no-follow reads.",
+ )
+ root_fd = _open_absolute_directory_posix(self.root)
+ try:
+ opened = os.fstat(root_fd)
+ if not stat.S_ISDIR(opened.st_mode) or not os.path.samestat(metadata, opened):
+ raise SecurePathError("unsafe_root", "Secure root changed while being opened.")
+ except BaseException:
+ os.close(root_fd)
+ raise
+ self._root_fd = root_fd
+ self._entered = True
+ return self
+
+ if os.name == "nt":
+ self._windows_root_handles = _windows_open_anchored_directory_chain(self.root, expected=metadata)
+ self._entered = True
+ return self
+
+ raise SecurePathError("secure_open_unavailable", "This platform cannot guarantee no-follow reads.")
+
+ def __exit__(self, _exc_type, _exc, _traceback) -> None:
+ if self._root_fd is not None:
+ os.close(self._root_fd)
+ self._root_fd = None
+ while self._windows_root_handles:
+ _windows_close_handle(self._windows_root_handles.pop())
+ self._entered = False
+
+ def read_bytes(
+ self,
+ relative_path: Path,
+ max_bytes: int,
+ *,
+ expected: os.stat_result | None = None,
+ ) -> tuple[bytes, os.stat_result]:
+ """Read one bounded regular single-link file without following redirects."""
+ if not self._entered:
+ raise SecurePathError("secure_open_unavailable", "Secure root context is not active.")
+ relative_path = _relative_path(relative_path)
+ if max_bytes < 0:
+ raise ValueError("max_bytes must be non-negative")
+ if os.name == "posix":
+ descriptor = self._open_posix(relative_path, expected)
+ elif os.name == "nt":
+ descriptor = self._open_windows(relative_path, expected)
+ else:
+ raise SecurePathError("secure_open_unavailable", "Secure no-follow reads are unavailable.")
+
+ try:
+ opened = os.fstat(descriptor)
+ _validate_opened_file(opened, relative_path, expected)
+ if opened.st_size > max_bytes:
+ raise SecurePathError(
+ "file_size_limit",
+ f"Selected file exceeds the {max_bytes}-byte limit: {relative_path.as_posix()}",
+ relative_path=relative_path.as_posix(),
+ metadata={"actual_bytes": opened.st_size, "limit_bytes": max_bytes},
+ )
+ chunks: list[bytes] = []
+ total = 0
+ while True:
+ chunk = os.read(descriptor, min(65_536, max_bytes + 1 - total))
+ if not chunk:
+ break
+ chunks.append(chunk)
+ total += len(chunk)
+ if total > max_bytes:
+ raise SecurePathError(
+ "file_size_limit",
+ f"Selected file exceeds the {max_bytes}-byte limit: {relative_path.as_posix()}",
+ relative_path=relative_path.as_posix(),
+ metadata={"actual_bytes": total, "limit_bytes": max_bytes},
+ )
+ after = os.fstat(descriptor)
+ _validate_opened_file(after, relative_path, opened)
+ return b"".join(chunks), opened
+ finally:
+ os.close(descriptor)
+
+ def _open_posix(self, relative_path: Path, expected: os.stat_result | None) -> int:
+ if self._root_fd is None:
+ raise SecurePathError("secure_open_unavailable", "Secure root descriptor is unavailable.")
+ directory_fd = os.dup(self._root_fd)
+ directory_flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0)
+ file_flags = (
+ os.O_RDONLY
+ | os.O_NOFOLLOW
+ | getattr(os, "O_CLOEXEC", 0)
+ | getattr(os, "O_BINARY", 0)
+ | getattr(os, "O_NONBLOCK", 0)
+ | getattr(os, "O_NOCTTY", 0)
+ )
+ try:
+ for component in relative_path.parts[:-1]:
+ try:
+ child_fd = os.open(component, directory_flags, dir_fd=directory_fd)
+ except OSError as exc:
+ raise SecurePathError(
+ "unsafe_path",
+ f"Cannot securely traverse path component {component!r}: {exc}",
+ relative_path=relative_path.as_posix(),
+ ) from exc
+ try:
+ child_metadata = os.fstat(child_fd)
+ except BaseException:
+ os.close(child_fd)
+ raise
+ if not stat.S_ISDIR(child_metadata.st_mode) or stat_is_link_or_reparse(child_metadata):
+ os.close(child_fd)
+ raise SecurePathError(
+ "unsafe_path",
+ f"Path component is not a regular directory: {component}",
+ relative_path=relative_path.as_posix(),
+ )
+ os.close(directory_fd)
+ directory_fd = child_fd
+
+ try:
+ before = os.stat(relative_path.name, dir_fd=directory_fd, follow_symlinks=False)
+ except OSError as exc:
+ raise SecurePathError(
+ "unsafe_path",
+ f"Cannot inspect selected file securely: {relative_path.as_posix()}: {exc}",
+ relative_path=relative_path.as_posix(),
+ ) from exc
+ _validate_opened_file(before, relative_path, expected)
+ try:
+ descriptor = os.open(relative_path.name, file_flags, dir_fd=directory_fd)
+ except OSError as exc:
+ message = "Selected path is a symlink or unsafe file" if exc.errno == errno.ELOOP else str(exc)
+ raise SecurePathError(
+ "unsafe_path",
+ f"Cannot securely open {relative_path.as_posix()}: {message}",
+ relative_path=relative_path.as_posix(),
+ ) from exc
+ try:
+ _validate_opened_file(os.fstat(descriptor), relative_path, before)
+ except BaseException:
+ os.close(descriptor)
+ raise
+ return descriptor
+ finally:
+ os.close(directory_fd)
+
+ def _open_windows(self, relative_path: Path, expected: os.stat_result | None) -> int:
+ if not self._windows_root_handles:
+ raise SecurePathError("secure_open_unavailable", "Secure root handle is unavailable.")
+
+ import msvcrt
+
+ directory_handles: list[int] = []
+ parent_handle = self._windows_root_handles[-1]
+ descriptor = -1
+ native_file_handle = -1
+ try:
+ for component in relative_path.parts[:-1]:
+ native_directory_handle = _windows_open_relative_handle(
+ parent_handle,
+ component,
+ access=_WINDOWS_DIRECTORY_READ_ACCESS,
+ share=_WINDOWS_SHARE_READ_WRITE,
+ disposition=_WINDOWS_FILE_OPEN,
+ file_attributes=0,
+ create_options=_WINDOWS_DIRECTORY_OPEN_OPTIONS,
+ )
+ try:
+ _validate_windows_read_directory_handle(native_directory_handle, relative_path)
+ except BaseException:
+ _windows_close_handle(native_directory_handle)
+ raise
+ directory_handles.append(native_directory_handle)
+ parent_handle = native_directory_handle
+
+ native_file_handle = _windows_open_relative_handle(
+ parent_handle,
+ relative_path.name,
+ access=_WINDOWS_FILE_READ_ACCESS,
+ # Deny write/delete sharing for the selected file while the
+ # CRT descriptor is live. This closes same-size rewrite races
+ # that Windows creation-time metadata cannot detect.
+ share=_WINDOWS_SHARE_READ,
+ disposition=_WINDOWS_FILE_OPEN,
+ file_attributes=0,
+ create_options=_WINDOWS_FILE_OPEN_OPTIONS,
+ )
+ _validate_windows_read_file_handle(native_file_handle, relative_path)
+ descriptor = msvcrt.open_osfhandle(
+ native_file_handle,
+ os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOINHERIT", 0),
+ )
+ native_file_handle = -1
+ opened = os.fstat(descriptor)
+ _validate_opened_file(opened, relative_path, expected)
+ return descriptor
+ except OSError as exc:
+ if descriptor >= 0:
+ os.close(descriptor)
+ descriptor = -1
+ raise SecurePathError(
+ "unsafe_path",
+ f"Cannot securely open selected file {relative_path.as_posix()}: {exc}",
+ relative_path=relative_path.as_posix(),
+ ) from exc
+ except BaseException:
+ if descriptor >= 0:
+ os.close(descriptor)
+ raise
+ finally:
+ if native_file_handle >= 0:
+ _windows_close_handle(native_file_handle)
+ while directory_handles:
+ _windows_close_handle(directory_handles.pop())
+
+
+def _windows_kernel32():
+ if os.name != "nt":
+ raise OSError("Windows handle operations are unavailable on this platform")
+ import ctypes
+
+ return ctypes.WinDLL("kernel32", use_last_error=True)
+
+
+def _windows_raise_last_error(message: str) -> OSError:
+ import ctypes
+
+ error = ctypes.get_last_error()
+ return OSError(error, message)
+
+
+def _windows_open_handle(
+ path: Path,
+ *,
+ access: int,
+ share: int,
+ disposition: int,
+ flags: int,
+) -> int:
+ import ctypes
+ from ctypes import wintypes
+
+ kernel32 = _windows_kernel32()
+ create_file = kernel32.CreateFileW
+ create_file.argtypes = [
+ wintypes.LPCWSTR,
+ wintypes.DWORD,
+ wintypes.DWORD,
+ wintypes.LPVOID,
+ wintypes.DWORD,
+ wintypes.DWORD,
+ wintypes.HANDLE,
+ ]
+ create_file.restype = wintypes.HANDLE
+ handle = create_file(os.fspath(path), access, share, None, disposition, flags, None)
+ invalid_handle = ctypes.c_void_p(-1).value
+ if handle == invalid_handle:
+ raise _windows_raise_last_error(f"Cannot open Windows filesystem handle: {path}")
+ return int(handle)
+
+
+def _validate_windows_path_component(name: str, *, label: str) -> None:
+ """Reject Win32 normalization aliases, device names, ADS, and invalid UTF-16."""
+ invalid_characters = '<>:"/\\|?*'
+ stem = name.split(".", 1)[0].rstrip(" .").casefold()
+ reserved = {
+ "con",
+ "prn",
+ "aux",
+ "nul",
+ *(f"com{index}" for index in range(1, 10)),
+ *(f"lpt{index}" for index in range(1, 10)),
+ *(f"com{index}" for index in "¹²³"),
+ *(f"lpt{index}" for index in "¹²³"),
+ }
+ try:
+ utf16_units = len(name.encode("utf-16-le")) // 2
+ except UnicodeEncodeError as exc:
+ raise SecurePathError("unsafe_path", f"{label} has an unsafe Windows file name.") from exc
+ if (
+ not name
+ or name in {".", ".."}
+ or utf16_units > 255
+ or any(ord(character) < 32 or character in invalid_characters for character in name)
+ or name.endswith((" ", "."))
+ or stem in reserved
+ ):
+ raise SecurePathError("unsafe_path", f"{label} has an unsafe Windows file name.")
+
+
+def _windows_open_relative_handle(
+ parent_handle: int,
+ name: str,
+ *,
+ access: int,
+ share: int,
+ disposition: int,
+ file_attributes: int,
+ create_options: int,
+ object_attributes_flags: int = _WINDOWS_OBJECT_ATTRIBUTES_FLAGS,
+) -> int:
+ """Open one path component relative to a held native directory handle."""
+ import ctypes
+ from ctypes import wintypes
+
+ _validate_windows_path_component(name, label="Anchored path component")
+
+ class _UnicodeString(ctypes.Structure):
+ _fields_ = [
+ ("Length", wintypes.USHORT),
+ ("MaximumLength", wintypes.USHORT),
+ ("Buffer", wintypes.LPWSTR),
+ ]
+
+ class _ObjectAttributes(ctypes.Structure):
+ _fields_ = [
+ ("Length", wintypes.ULONG),
+ ("RootDirectory", wintypes.HANDLE),
+ ("ObjectName", ctypes.POINTER(_UnicodeString)),
+ ("Attributes", wintypes.ULONG),
+ ("SecurityDescriptor", wintypes.LPVOID),
+ ("SecurityQualityOfService", wintypes.LPVOID),
+ ]
+
+ class _IoStatusValue(ctypes.Union):
+ _fields_ = [("Status", wintypes.LONG), ("Pointer", wintypes.LPVOID)] # noqa: RUF012
+
+ class _IoStatusBlock(ctypes.Structure):
+ _fields_ = [("Value", _IoStatusValue), ("Information", ctypes.c_size_t)]
+
+ encoded_name = name.encode("utf-16-le")
+ name_buffer = ctypes.create_unicode_buffer(name)
+ unicode_name = _UnicodeString(
+ Length=len(encoded_name),
+ MaximumLength=len(encoded_name) + ctypes.sizeof(wintypes.WCHAR),
+ Buffer=ctypes.cast(name_buffer, wintypes.LPWSTR),
+ )
+ object_attributes = _ObjectAttributes(
+ Length=ctypes.sizeof(_ObjectAttributes),
+ RootDirectory=parent_handle,
+ ObjectName=ctypes.pointer(unicode_name),
+ Attributes=object_attributes_flags,
+ SecurityDescriptor=None,
+ SecurityQualityOfService=None,
+ )
+ io_status = _IoStatusBlock()
+ handle = wintypes.HANDLE()
+
+ ntdll = ctypes.WinDLL("ntdll")
+ nt_create_file = ntdll.NtCreateFile
+ nt_create_file.argtypes = [
+ ctypes.POINTER(wintypes.HANDLE),
+ wintypes.DWORD,
+ ctypes.POINTER(_ObjectAttributes),
+ ctypes.POINTER(_IoStatusBlock),
+ wintypes.LPVOID,
+ wintypes.ULONG,
+ wintypes.ULONG,
+ wintypes.ULONG,
+ wintypes.ULONG,
+ wintypes.LPVOID,
+ wintypes.ULONG,
+ ]
+ nt_create_file.restype = wintypes.LONG
+ status = int(
+ nt_create_file(
+ ctypes.byref(handle),
+ access,
+ ctypes.byref(object_attributes),
+ ctypes.byref(io_status),
+ None,
+ file_attributes,
+ share,
+ disposition,
+ create_options,
+ None,
+ 0,
+ )
+ )
+ if status < 0:
+ rtl_status_to_error = ntdll.RtlNtStatusToDosError
+ rtl_status_to_error.argtypes = [wintypes.LONG]
+ rtl_status_to_error.restype = wintypes.ULONG
+ error = int(rtl_status_to_error(status))
+ raise OSError(error, f"Cannot open anchored Windows path component: {name}")
+ if not handle.value:
+ raise OSError("NtCreateFile succeeded without returning a file handle")
+ return int(handle.value)
+
+
+def _windows_close_handle(handle: int) -> None:
+ from ctypes import wintypes
+
+ kernel32 = _windows_kernel32()
+ close_handle = kernel32.CloseHandle
+ close_handle.argtypes = [wintypes.HANDLE]
+ close_handle.restype = wintypes.BOOL
+ if not close_handle(handle):
+ raise _windows_raise_last_error("Cannot close Windows filesystem handle")
+
+
+def _windows_handle_metadata(handle: int) -> _WindowsHandleMetadata:
+ import ctypes
+ from ctypes import wintypes
+
+ class _ByHandleFileInformation(ctypes.Structure):
+ _fields_ = [
+ ("dwFileAttributes", wintypes.DWORD),
+ ("ftCreationTime", wintypes.FILETIME),
+ ("ftLastAccessTime", wintypes.FILETIME),
+ ("ftLastWriteTime", wintypes.FILETIME),
+ ("dwVolumeSerialNumber", wintypes.DWORD),
+ ("nFileSizeHigh", wintypes.DWORD),
+ ("nFileSizeLow", wintypes.DWORD),
+ ("nNumberOfLinks", wintypes.DWORD),
+ ("nFileIndexHigh", wintypes.DWORD),
+ ("nFileIndexLow", wintypes.DWORD),
+ ]
+
+ kernel32 = _windows_kernel32()
+ get_information = kernel32.GetFileInformationByHandle
+ get_information.argtypes = [wintypes.HANDLE, ctypes.POINTER(_ByHandleFileInformation)]
+ get_information.restype = wintypes.BOOL
+ information = _ByHandleFileInformation()
+ if not get_information(handle, ctypes.byref(information)):
+ raise _windows_raise_last_error("Cannot inspect open Windows filesystem handle")
+ return _WindowsHandleMetadata(
+ attributes=int(information.dwFileAttributes),
+ volume_serial=int(information.dwVolumeSerialNumber),
+ file_id=(int(information.nFileIndexHigh) << 32) | int(information.nFileIndexLow),
+ size=(int(information.nFileSizeHigh) << 32) | int(information.nFileSizeLow),
+ link_count=int(information.nNumberOfLinks),
+ last_write_time=(int(information.ftLastWriteTime.dwHighDateTime) << 32)
+ | int(information.ftLastWriteTime.dwLowDateTime),
+ )
+
+
+def _validate_windows_read_directory_handle(handle: int, relative_path: Path) -> _WindowsHandleMetadata:
+ """Require one opened Windows traversal component to be a plain directory."""
+ metadata = _windows_handle_metadata(handle)
+ directory_attribute = 0x10
+ reparse_attribute = 0x400
+ if metadata.attributes & reparse_attribute or not metadata.attributes & directory_attribute:
+ raise SecurePathError(
+ "unsafe_path",
+ f"Path contains a non-directory or reparse component: {relative_path.as_posix()}",
+ relative_path=relative_path.as_posix(),
+ )
+ return metadata
+
+
+def _validate_windows_read_file_handle(handle: int, relative_path: Path) -> _WindowsHandleMetadata:
+ """Require one selected Windows handle to be regular, single-link, and no-follow."""
+ metadata = _windows_handle_metadata(handle)
+ directory_attribute = 0x10
+ reparse_attribute = 0x400
+ if metadata.attributes & (directory_attribute | reparse_attribute):
+ raise SecurePathError(
+ "unsafe_path",
+ f"Refusing selected directory or reparse point: {relative_path.as_posix()}",
+ relative_path=relative_path.as_posix(),
+ )
+ if metadata.link_count != 1:
+ _raise_unsafe_file(relative_path, hardlink=True)
+ return metadata
+
+
+def _windows_open_anchored_directory_chain(
+ path: Path,
+ *,
+ expected: os.stat_result,
+) -> list[int]:
+ """Pin an absolute directory from its volume anchor without following reparses."""
+ absolute = _absolute_no_resolve(path)
+ if not absolute.anchor:
+ raise SecurePathError("unsafe_root", "Windows root has no filesystem anchor.")
+
+ anchor = Path(absolute.anchor)
+ handles: list[int] = []
+ try:
+ anchor_handle = _windows_open_handle(
+ anchor,
+ access=_WINDOWS_DIRECTORY_READ_ACCESS,
+ share=_WINDOWS_SHARE_READ_WRITE,
+ disposition=3,
+ flags=0x02000000 | _WINDOWS_FILE_OPEN_REPARSE_POINT,
+ )
+ handles.append(anchor_handle)
+ _validate_windows_read_directory_handle(anchor_handle, anchor)
+
+ current_path = anchor
+ parent_handle = anchor_handle
+ for component in absolute.parts[1:]:
+ current_path /= component
+ child_handle = _windows_open_relative_handle(
+ parent_handle,
+ component,
+ access=_WINDOWS_DIRECTORY_READ_ACCESS,
+ share=_WINDOWS_SHARE_READ_WRITE,
+ disposition=_WINDOWS_FILE_OPEN,
+ file_attributes=0,
+ create_options=_WINDOWS_DIRECTORY_OPEN_OPTIONS,
+ )
+ handles.append(child_handle)
+ _validate_windows_read_directory_handle(child_handle, current_path)
+ parent_handle = child_handle
+
+ try:
+ declared = absolute.lstat()
+ except OSError as exc:
+ raise SecurePathError("unsafe_root", f"Cannot revalidate declared root: {exc}") from exc
+ if stat_is_link_or_reparse(declared) or not stat.S_ISDIR(declared.st_mode):
+ raise SecurePathError("unsafe_root", "Declared root became a reparse point or non-directory.")
+ if not os.path.samestat(expected, declared):
+ raise SecurePathError("unsafe_root", "Root changed identity while native handles were opened.")
+ return handles
+ except BaseException:
+ while handles:
+ _windows_close_handle(handles.pop())
+ raise
diff --git a/tests/reporting/test_reporters.py b/tests/reporting/test_reporters.py
index c0702afe..76bf4f25 100644
--- a/tests/reporting/test_reporters.py
+++ b/tests/reporting/test_reporters.py
@@ -702,6 +702,7 @@ class TestTier3HarborCanonicalPayload:
),
encoding="utf-8",
)
+ (run_dir / "run_config.json").write_text("{}", encoding="utf-8")
(run_dir / "result.json").write_text(json.dumps({"run_id": run_id}), encoding="utf-8")
(results_root / "latest").symlink_to(run_id)
diff --git a/tests/reporting/test_unified_tier3_report.py b/tests/reporting/test_unified_tier3_report.py
index f8126e96..1b91972c 100644
--- a/tests/reporting/test_unified_tier3_report.py
+++ b/tests/reporting/test_unified_tier3_report.py
@@ -53,6 +53,75 @@ def _write_summary(
)
+def _write_authenticated_pre_status_summary(run_dir: Path, *, score: float = 0.8) -> None:
+ summary = run_dir / "opencode" / "with-skill" / "summary.json"
+ summary.parent.mkdir(parents=True)
+ (run_dir / "run_config.json").write_text(
+ json.dumps(
+ {
+ "harbor": {"n_attempts": {"value": 1, "source": "ACES default"}},
+ "task_source": "evals_json",
+ "agents": {
+ "opencode": {
+ "agent": "opencode",
+ "model": "test-model",
+ "source": "test default",
+ "occurrence": "1",
+ }
+ },
+ }
+ ),
+ encoding="utf-8",
+ )
+ summary.write_text(
+ json.dumps(
+ {
+ "agent": "opencode",
+ "model": "test-model",
+ "model_source": "test default",
+ "scores": {"security": score},
+ "custom_scores": {},
+ "metric_set": "aces-default-v2",
+ "metrics": ["security"],
+ "dimensions": {"safety": {"score": score, "sources": {"security": 1.0}}},
+ "num_trials": 1,
+ "pass_at_k": {
+ "k": 1,
+ "pass_threshold": 0.5,
+ "stop_on_pass": False,
+ "passed_cases": 1,
+ "failed_cases": 0,
+ "total_cases": 1,
+ "rate": 1.0,
+ "attempts_used": 1,
+ "max_attempts_possible": 1,
+ "avg_attempts_used": 1.0,
+ "extra_cases": [],
+ "cases": {
+ "case": {
+ "passed": True,
+ "first_pass_attempt": 1,
+ "attempts_used": 1,
+ "attempts_skipped": 0,
+ "attempts_missing": 0,
+ "best_score": score,
+ "attempts": [
+ {
+ "attempt": 1,
+ "trial": "case__attempt1",
+ "score": score,
+ "passed": True,
+ }
+ ],
+ }
+ },
+ },
+ }
+ ),
+ encoding="utf-8",
+ )
+
+
def _render_agent_payload(payload: dict) -> str:
result = ValidationResult(validator_name="AGENT_EVAL", validator_description="Live evaluation")
result.metadata["agent_eval"] = payload
@@ -125,6 +194,21 @@ def test_standalone_tier3_uses_generic_tier3_only_report(tmp_path: Path) -> None
assert "SkillEvaluator" in html
+def test_authenticated_pre_status_rerender_preserves_historical_scores(tmp_path: Path) -> None:
+ skill = tmp_path / "demo"
+ skill.mkdir()
+ run_dir = tmp_path / "results" / "20260709_120008"
+ _write_authenticated_pre_status_summary(run_dir)
+
+ tier3 = agent_eval_result_from_directory(skill, run_dir, use_llm_judge=False)
+
+ assert tier3 is not None
+ payload = tier3.metadata["agent_eval"]
+ assert payload["execution_status"] == "succeeded"
+ assert payload["overall_score"] is not None
+ assert payload["agents"]["opencode"]["evaluators"]["security"]["with_skill"] == 0.8
+
+
def test_canonical_report_prefers_agentskills_dataset_fields() -> None:
payload = build_agent_eval_payload(
"hld-documents",
@@ -866,6 +950,7 @@ def test_view_regenerates_missing_report_with_canonical_renderer(monkeypatch, tm
results_root = tmp_path / "results"
run_dir = results_root / skill.name / "20260709_120002"
run_dir.mkdir(parents=True)
+ (run_dir / "run_config.json").write_text("{}", encoding="utf-8")
(run_dir / "result.json").write_text(
json.dumps({"run_id": run_dir.name}),
encoding="utf-8",
diff --git a/tests/test_evaluation_service.py b/tests/test_evaluation_service.py
index 5e8b9812..59c6eaf9 100644
--- a/tests/test_evaluation_service.py
+++ b/tests/test_evaluation_service.py
@@ -664,6 +664,7 @@ def test_report_discovery_handles_timestamped_results(tmp_path: Path) -> None:
run_dir = results_root / "myskill" / "2026-06-18_120000"
run_dir.mkdir(parents=True)
(run_dir / "report.html").write_text("<html></html>", encoding="utf-8")
+ (run_dir / "run_config.json").write_text("{}", encoding="utf-8")
(run_dir / "result.json").write_text(json.dumps({"run_id": run_dir.name}), encoding="utf-8")
(results_root / "myskill" / "latest").symlink_to(run_dir.name)
diff --git a/tests/test_harbor_adapter_safety.py b/tests/test_harbor_adapter_safety.py
index fe0c5b0e..64756099 100644
--- a/tests/test_harbor_adapter_safety.py
+++ b/tests/test_harbor_adapter_safety.py
@@ -356,7 +356,7 @@ def test_custom_grader_candidates_reject_external_symlinks(tmp_path: Path, candi
_symlink_or_skip(grader, outside)
task_dir = tmp_path / "task"
- with pytest.raises(ValueError, match="non-symlinked regular file contained under evals/"):
+ with pytest.raises(ValueError, match=r"non-symlinked regular file contained under evals/|symlink|reparse"):
_copy_custom_grader(task_dir, skill, "custom_only")
assert not (task_dir / "tests" / candidate.name).exists()
@@ -476,7 +476,7 @@ def test_task_modes_reject_custom_grader_symlink_before_copy(tmp_path: Path, tas
_symlink_or_skip(skill / "evals" / "grader.py", outside)
output_dir = tmp_path / "tasks"
- with pytest.raises(ValueError, match="non-symlinked regular file contained under evals/"):
+ with pytest.raises(ValueError, match=r"non-symlinked regular file contained under evals/|symlink|reparse"):
if task_source == "generated":
generate_harbor_tasks(skill, output_dir, with_skill=False, grading_mode="custom_only")
else:
diff --git a/tests/test_harbor_html_report_failures.py b/tests/test_harbor_html_report_failures.py
index 96bd331d..a548f09a 100644
--- a/tests/test_harbor_html_report_failures.py
+++ b/tests/test_harbor_html_report_failures.py
@@ -523,7 +523,8 @@ def test_html_generation_failure_is_persisted_identically_to_returned_result(
) -> None:
"""A report warning is part of the final result contract on disk and in memory."""
skill_path = tmp_path / "demo"
- skill_path.mkdir()
+ (skill_path / "evals").mkdir(parents=True)
+ (skill_path / "evals" / "evals.json").write_text("[]\n", encoding="utf-8")
cli_results_dir = tmp_path / "results"
output_dir = external_results_root(cli_results_dir, skill_path)
provider = ProviderConfig(
diff --git a/tests/test_harbor_input_staging.py b/tests/test_harbor_input_staging.py
index 29f202cb..65d8edd7 100644
--- a/tests/test_harbor_input_staging.py
+++ b/tests/test_harbor_input_staging.py
@@ -18,8 +18,10 @@ from pathlib import Path
import pytest
+import skillevaluator.tier3.harbor.adapter as adapter_module
from skillevaluator.tier3.harbor.adapter import (
_entry_file_refs,
+ _load_mcp_servers,
_resolve_entry_file_ref,
_stage_task_inputs,
generate_harbor_tasks,
@@ -281,6 +283,68 @@ def test_generated_tasks_apply_per_entry_input_isolation(tmp_path: Path):
assert "COPY input/ /workspace/input/" not in dockerfile
+def test_generation_uses_one_private_evals_snapshot(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ skill, evals, _ = _make_skill(tmp_path)
+ (skill / "SKILL.md").write_text("# Snapshot test\n", encoding="utf-8")
+ (evals / "evals.json").write_text(
+ json.dumps([{"id": "case", "question": "dataset-A"}]),
+ encoding="utf-8",
+ )
+ (evals / "files" / "global.txt").write_text("fixture-A", encoding="utf-8")
+ moved = tmp_path / "moved-evals"
+ real_load = adapter_module._load_evals
+ swapped = False
+
+ def swap_original_evals_after_dataset_read(path: Path) -> list[dict[str, object]]:
+ nonlocal swapped
+ entries = real_load(path)
+ if not swapped:
+ swapped = True
+ evals.rename(moved)
+ replacement_files = evals / "files"
+ replacement_files.mkdir(parents=True)
+ (evals / "evals.json").write_text(
+ json.dumps([{"id": "case", "question": "dataset-B"}]),
+ encoding="utf-8",
+ )
+ (replacement_files / "global.txt").write_text("fixture-B", encoding="utf-8")
+ return entries
+
+ monkeypatch.setattr(adapter_module, "_load_evals", swap_original_evals_after_dataset_read)
+
+ task = generate_harbor_tasks(skill, tmp_path / "generated", with_skill=False)[0]
+
+ assert (task / "instruction.md").read_text(encoding="utf-8") == "dataset-A\n"
+ assert (task / "environment" / "input" / "global.txt").read_text(encoding="utf-8") == "fixture-A"
+
+
[email protected]("results_name", ("results", "Results"))
+def test_private_evals_snapshot_ignores_results_through_platform_root_aliases(
+ tmp_path: Path,
+ results_name: str,
+) -> None:
+ skill, evals, _ = _make_skill(tmp_path)
+ (skill / "SKILL.md").write_text("# Snapshot test\n", encoding="utf-8")
+ (evals / "evals.json").write_text(
+ json.dumps([{"id": "case", "question": "evaluate"}]),
+ encoding="utf-8",
+ )
+ results_dir = evals / results_name
+ run_dir = results_dir / "run-1"
+ run_dir.mkdir(parents=True)
+ try:
+ (results_dir / "current").symlink_to(run_dir.name, target_is_directory=True)
+ except OSError as exc: # pragma: no cover - host policy, primarily native Windows
+ pytest.skip(f"symlinks unavailable on this host: {exc}")
+
+ task = generate_harbor_tasks(skill, tmp_path / "generated", with_skill=False)[0]
+
+ assert task.name == "case"
+
+
@pytest.mark.parametrize("base_image", ["", "example.invalid/eval-base:latest"])
def test_generated_task_copies_explicit_ref_without_shared_files_directory(tmp_path: Path, base_image: str):
skill, evals, _ = _make_skill(tmp_path)
@@ -316,3 +380,260 @@ class TestResolveEntryFileRef:
skill, evals, _ = _make_skill(tmp_path)
with pytest.raises(ValueError, match="unsupported URI scheme"):
_resolve_entry_file_ref("https://example.com/x", skill_path=skill, evals_dir=evals, input_files_dir=None)
+
+ @pytest.mark.parametrize(
+ "relative_path,is_directory",
+ [
+ ("evals.json", False),
+ ("EVALS.JSON", False),
+ ("dataset.yaml", False),
+ ("config.yml", False),
+ ("grader.py", False),
+ ("grader.sh", False),
+ ("EVAL.md", False),
+ ("benchmark_" + "conversion_report.md", False),
+ (".skillevaluator-generated-output", False),
+ ("tests", True),
+ ("harbor", True),
+ ("environment", True),
+ ("results", True),
+ ],
+ )
+ def test_evaluator_only_assets_cannot_be_declared_as_task_input(
+ self,
+ tmp_path: Path,
+ relative_path: str,
+ is_directory: bool,
+ ) -> None:
+ skill, evals, env_dir = _make_skill(tmp_path)
+ source = evals / relative_path
+ if is_directory:
+ source.mkdir()
+ (source / "secret.txt").write_text("evaluator-only", encoding="utf-8")
+ else:
+ source.write_text("evaluator-only", encoding="utf-8")
+
+ with pytest.raises(ValueError, match="evaluator-only"):
+ _stage_task_inputs(
+ env_dir,
+ input_files_dir=evals / "files",
+ entry={"id": "case", "files": [f"evals/{relative_path}"]},
+ source_skill_path=skill,
+ evals_dir=evals,
+ )
+
+ assert not (env_dir / "input").exists()
+
+
+class TestLoadMcpServersSafety:
+ def test_linked_mcp_configuration_is_rejected(self, tmp_path: Path) -> None:
+ skill = tmp_path / "skill"
+ environment = skill / "evals" / "environment"
+ environment.mkdir(parents=True)
+ outside = tmp_path / "outside-mcp_servers.toml"
+ outside.write_text('[[mcp_servers]]\nname = "outside"\ncommand = "/bin/sh"\n', encoding="utf-8")
+ link = environment / "mcp_servers.toml"
+ try:
+ link.symlink_to(outside)
+ except OSError as exc: # pragma: no cover - host policy, primarily native Windows
+ pytest.skip(f"symlinks unavailable on this host: {exc}")
+
+ with pytest.raises(ValueError, match="regular non-linked file"):
+ _load_mcp_servers(skill)
+
+ def test_hardlinked_mcp_configuration_is_rejected(self, tmp_path: Path) -> None:
+ skill = tmp_path / "skill"
+ environment = skill / "evals" / "environment"
+ environment.mkdir(parents=True)
+ outside = tmp_path / "outside-mcp.toml"
+ outside.write_text('[[mcp_servers]]\nname = "outside"\ncommand = "/bin/sh"\n', encoding="utf-8")
+ try:
+ os.link(outside, environment / "mcp_servers.toml")
+ except OSError as exc: # pragma: no cover - filesystem policy
+ pytest.skip(f"hardlinks unavailable on this host: {exc}")
+
+ with pytest.raises(ValueError, match="regular non-linked file"):
+ _load_mcp_servers(skill)
+
+ def test_linked_mcp_parent_directory_is_rejected(self, tmp_path: Path) -> None:
+ skill = tmp_path / "skill"
+ evals = skill / "evals"
+ evals.mkdir(parents=True)
+ outside = tmp_path / "outside-environment"
+ outside.mkdir()
+ (outside / "mcp_servers.toml").write_text(
+ '[[mcp_servers]]\nname = "outside"\ncommand = "/bin/sh"\n',
+ encoding="utf-8",
+ )
+ try:
+ (evals / "environment").symlink_to(outside, target_is_directory=True)
+ except OSError as exc: # pragma: no cover - host policy, primarily native Windows
+ pytest.skip(f"symlinks unavailable on this host: {exc}")
+
+ with pytest.raises(ValueError, match=r"symlink|reparse"):
+ _load_mcp_servers(skill)
+
+ def test_linked_mcp_parent_is_rejected_even_when_config_is_absent(self, tmp_path: Path) -> None:
+ skill = tmp_path / "skill"
+ evals = skill / "evals"
+ evals.mkdir(parents=True)
+ outside = tmp_path / "outside-environment"
+ outside.mkdir()
+ try:
+ (evals / "environment").symlink_to(outside, target_is_directory=True)
+ except OSError as exc: # pragma: no cover - host policy, primarily native Windows
+ pytest.skip(f"symlinks unavailable on this host: {exc}")
+
+ with pytest.raises(ValueError, match=r"symlink|reparse"):
+ _load_mcp_servers(skill)
+
+ def test_mcp_parent_change_is_rejected_before_missing_config_return(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ ) -> None:
+ skill = tmp_path / "skill"
+ environment = skill / "evals" / "environment"
+ environment.mkdir(parents=True)
+ config = environment / "mcp_servers.toml"
+ config.write_text('[[mcp_servers]]\nname = "original"\ncommand = "/bin/sh"\n', encoding="utf-8")
+ moved = tmp_path / "moved-environment"
+ real_lexists = adapter_module.os.path.lexists
+ swapped = False
+
+ def swap_before_missing_check(path: os.PathLike[str] | str) -> bool:
+ nonlocal swapped
+ if not swapped and Path(path) == config:
+ swapped = True
+ environment.rename(moved)
+ environment.mkdir()
+ return False
+ return real_lexists(path)
+
+ monkeypatch.setattr(adapter_module.os.path, "lexists", swap_before_missing_check)
+
+ with pytest.raises(ValueError, match="changed"):
+ _load_mcp_servers(skill)
+
+ def test_mcp_parent_replaced_before_open_is_rejected(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ ) -> None:
+ skill = tmp_path / "skill"
+ environment = skill / "evals" / "environment"
+ environment.mkdir(parents=True)
+ config = environment / "mcp_servers.toml"
+ config.write_text('[[mcp_servers]]\nname = "original"\ncommand = "/bin/sh"\n', encoding="utf-8")
+ moved = tmp_path / "moved-environment"
+ original_read_bytes = adapter_module.SecureRoot.read_bytes
+ swapped = False
+
+ def swap_parent(
+ secure_root: adapter_module.SecureRoot,
+ relative_path: Path,
+ max_bytes: int,
+ *,
+ expected: os.stat_result | None = None,
+ ) -> tuple[bytes, os.stat_result]:
+ nonlocal swapped
+ if not swapped and relative_path == Path("environment/mcp_servers.toml"):
+ swapped = True
+ environment.rename(moved)
+ environment.symlink_to(moved, target_is_directory=True)
+ return original_read_bytes(secure_root, relative_path, max_bytes, expected=expected)
+
+ monkeypatch.setattr(adapter_module.SecureRoot, "read_bytes", swap_parent)
+
+ with pytest.raises(ValueError, match=r"changed|symlink|reparse"):
+ _load_mcp_servers(skill)
+
+ def test_same_inode_mcp_rewrite_with_restored_mtime_is_rejected(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ ) -> None:
+ skill = tmp_path / "skill"
+ environment = skill / "evals" / "environment"
+ environment.mkdir(parents=True)
+ config = environment / "mcp_servers.toml"
+ original_payload = '[[mcp_servers]]\nname = "alpha"\ncommand = "/bin/sh"\n'
+ replacement_payload = '[[mcp_servers]]\nname = "bravo"\ncommand = "/bin/sh"\n'
+ assert len(original_payload) == len(replacement_payload)
+ config.write_text(original_payload, encoding="utf-8")
+ original_metadata = config.stat()
+ original_read = adapter_module.os.read
+ mutated = False
+
+ def rewrite_named_source(descriptor: int, size: int) -> bytes:
+ nonlocal mutated
+ payload = original_read(descriptor, size)
+ if not mutated:
+ mutated = True
+ config.write_text(replacement_payload, encoding="utf-8")
+ os.utime(
+ config,
+ ns=(original_metadata.st_atime_ns, original_metadata.st_mtime_ns),
+ )
+ mutated_metadata = config.stat()
+ assert mutated_metadata.st_ino == original_metadata.st_ino
+ assert mutated_metadata.st_size == original_metadata.st_size
+ assert mutated_metadata.st_mtime_ns == original_metadata.st_mtime_ns
+ return payload
+
+ monkeypatch.setattr(adapter_module.os, "read", rewrite_named_source)
+
+ with pytest.raises(ValueError, match="changed while it was read"):
+ _load_mcp_servers(skill)
+
+ def test_mcp_configuration_replaced_during_read_is_rejected(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ ) -> None:
+ skill = tmp_path / "skill"
+ environment = skill / "evals" / "environment"
+ environment.mkdir(parents=True)
+ config = environment / "mcp_servers.toml"
+ config.write_text('[[mcp_servers]]\nname = "original"\ncommand = "/bin/sh"\n', encoding="utf-8")
+ replacement = tmp_path / "replacement.toml"
+ replacement.write_text('[[mcp_servers]]\nname = "replaced"\ncommand = "/bin/sh"\n', encoding="utf-8")
+ original_read = adapter_module.os.read
+ swapped = False
+
+ def swap_named_source(descriptor: int, size: int) -> bytes:
+ nonlocal swapped
+ payload = original_read(descriptor, size)
+ if not swapped:
+ swapped = True
+ config.rename(environment / "original-opened.toml")
+ replacement.rename(config)
+ return payload
+
+ monkeypatch.setattr(adapter_module.os, "read", swap_named_source)
+
+ with pytest.raises(ValueError, match="changed while it was read"):
+ _load_mcp_servers(skill)
+
+ def test_task_generation_rejects_linked_mcp_before_output_mutation(self, tmp_path: Path) -> None:
+ skill = tmp_path / "skill"
+ evals = skill / "evals"
+ environment = evals / "environment"
+ environment.mkdir(parents=True)
+ (skill / "SKILL.md").write_text("# Test skill\n", encoding="utf-8")
+ (evals / "evals.json").write_text(
+ json.dumps([{"id": "case", "question": "Run the case"}]),
+ encoding="utf-8",
+ )
+ outside = tmp_path / "outside-mcp.toml"
+ outside.write_text('[[mcp_servers]]\nname = "outside"\ncommand = "/bin/sh"\n', encoding="utf-8")
+ try:
+ (environment / "mcp_servers.toml").symlink_to(outside)
+ except OSError as exc: # pragma: no cover - host policy, primarily native Windows
+ pytest.skip(f"symlinks unavailable on this host: {exc}")
+ output = tmp_path / "generated"
+
+ with pytest.raises(ValueError, match=r"regular non-linked|symlink|reparse"):
+ generate_harbor_tasks(skill, output, with_skill=False)
+
+ assert not output.exists()
diff --git a/tests/test_harbor_runner_environment.py b/tests/test_harbor_runner_environment.py
index 13549ac8..c8c76a95 100644
--- a/tests/test_harbor_runner_environment.py
+++ b/tests/test_harbor_runner_environment.py
@@ -388,7 +388,8 @@ def test_run_harbor_eval_stages_per_agent_credential_trees(
tmp_path,
) -> None:
skill = tmp_path / "demo"
- skill.mkdir()
+ (skill / "evals").mkdir(parents=True)
+ (skill / "evals" / "evals.json").write_text("[]\n", encoding="utf-8")
provider = _provider("nv_build")
emitted: list[tuple[str, bool, dict[str, str]]] = []
launched: dict[str, tuple[str, str, dict[str, str]]] = {}
@@ -459,7 +460,8 @@ def test_run_harbor_eval_rejects_provenance_key_inside_skill_before_creation(
tmp_path,
) -> None:
skill = tmp_path / "demo"
- skill.mkdir()
+ (skill / "evals").mkdir(parents=True)
+ (skill / "evals" / "evals.json").write_text("[]\n", encoding="utf-8")
key_path = skill / "private-output-provenance.key"
monkeypatch.setenv("SKILLEVALUATOR_OUTPUT_PROVENANCE_KEY_FILE", str(key_path))
monkeypatch.setattr(runner, "resolve_llm_provider", lambda: _provider("nv_build"))
diff --git a/tests/test_harbor_runtime_preflight.py b/tests/test_harbor_runtime_preflight.py
index 09e469d8..cbb0eb71 100644
--- a/tests/test_harbor_runtime_preflight.py
+++ b/tests/test_harbor_runtime_preflight.py
@@ -675,7 +675,8 @@ def test_runtime_preflight_failure_stops_full_matrix(monkeypatch, tmp_path: Path
from skillevaluator.tier3.harbor import runner
skill = tmp_path / "demo"
- skill.mkdir()
+ (skill / "evals").mkdir(parents=True)
+ (skill / "evals" / "evals.json").write_text("[]\n", encoding="utf-8")
provider = ProviderConfig(
provider="nv_build",
model="meta/llama-3.1-8b-instruct",
diff --git a/tests/test_harbor_runtime_skill_isolation.py b/tests/test_harbor_runtime_skill_isolation.py
index e8128fda..620d506a 100644
--- a/tests/test_harbor_runtime_skill_isolation.py
+++ b/tests/test_harbor_runtime_skill_isolation.py
@@ -23,6 +23,7 @@ from skillevaluator.tier3.harbor.adapter import (
_path_is_excluded,
generate_harbor_tasks,
prebuild_task_environments,
+ private_evaluator_skill_snapshot,
stage_native_harbor_tasks,
validate_results_root_location,
)
@@ -213,6 +214,55 @@ def test_native_tasks_project_runtime_skills_without_root_evals(tmp_path: Path)
assert (task / "tests" / "grader.py").is_file()
+def test_native_task_staging_uses_one_private_evals_snapshot(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ evals = target / "evals"
+ moved = tmp_path / "evals-A"
+ real_load = adapter_module._load_entries_by_id
+ swapped = False
+
+ def swap_original_evals_after_metadata_read(path: Path) -> dict[str, dict[str, object]]:
+ nonlocal swapped
+ entries = real_load(path)
+ if not swapped:
+ swapped = True
+ evals.rename(moved)
+ shutil.copytree(moved, evals)
+ (evals / "evals.json").write_text(
+ json.dumps(
+ [
+ {
+ "id": "case-001",
+ "question": "metadata-B",
+ "files": ["evals/files/selected.txt"],
+ }
+ ]
+ ),
+ encoding="utf-8",
+ )
+ (evals / "files" / "selected.txt").write_text("fixture-B\n", encoding="utf-8")
+ (evals / "harbor" / "case-001" / "instruction.md").write_text(
+ "native-B\n",
+ encoding="utf-8",
+ )
+ return entries
+
+ monkeypatch.setattr(adapter_module, "_load_entries_by_id", swap_original_evals_after_metadata_read)
+
+ task = stage_native_harbor_tasks(target, tmp_path / "native-snapshot", with_skill=False)[0]
+ entry = json.loads((task / "tests" / "entry.json").read_text(encoding="utf-8"))
+
+ assert swapped
+ assert (task / "instruction.md").read_text(encoding="utf-8") == "Run the native case.\n"
+ assert entry["question"] == "Use only the selected fixture."
+ assert (task / "environment" / "input" / "selected.txt").read_text(encoding="utf-8") == "SELECTED-FIXTURE\n"
+ assert (task / "tests" / "grader.py").is_file()
+
+
def test_base_image_dependency_collection_ignores_only_root_evals(tmp_path: Path) -> None:
_, target, references_dir, workspace = _write_projection_fixture(tmp_path)
@@ -365,7 +415,7 @@ def test_copy_repo_rejects_hardlink_alias(tmp_path: Path) -> None:
except OSError as exc: # pragma: no cover - filesystem policy
pytest.skip(f"hardlinks unavailable on this host: {exc}")
- with pytest.raises(ValueError, match=r"hardlink|hard link"):
+ with pytest.raises(ValueError, match=r"hard.?link"):
generate_harbor_tasks(
target,
tmp_path / "generated-hardlink",
@@ -1416,7 +1466,8 @@ def test_readonly_custom_environment_snapshot_stays_outside_docker_context(
original_copytree = adapter_module.copytree_secure
def record_custom_snapshot(source: Path, destination: Path, *args: object, **kwargs: object) -> None:
- if Path(source) == custom_environment:
+ source_path = Path(source)
+ if source_path.name == "environment" and source_path.parent.name == "evals":
snapshot_destinations.append(Path(destination))
original_copytree(source, destination, *args, **kwargs)
@@ -2672,3 +2723,350 @@ def test_eval_dataset_file_must_be_regular_and_unlinked(tmp_path: Path, kind: st
with pytest.raises(ValueError, match="regular non-linked file"):
generate_harbor_tasks(target, tmp_path / f"unsafe-dataset-{kind}")
+
+
[email protected]("kind", ["symlink", "hardlink", "fifo"])
+def test_selective_evaluator_snapshot_does_not_copy_unselected_unsafe_fixture(
+ tmp_path: Path,
+ kind: str,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ files = target / "evals" / "files"
+ unselected = files / "unselected-unsafe"
+ outside = tmp_path / "outside-unselected.txt"
+ outside.write_text("must not be inspected or copied\n", encoding="utf-8")
+ try:
+ if kind == "symlink":
+ unselected.symlink_to(outside)
+ elif kind == "hardlink":
+ unselected.hardlink_to(outside)
+ elif hasattr(os, "mkfifo"):
+ os.mkfifo(unselected)
+ else:
+ pytest.skip("FIFO creation is unavailable")
+ except OSError as exc:
+ pytest.skip(f"{kind} creation is unavailable: {exc}")
+
+ # A sparse large sibling models the ENOSPC/performance regression without
+ # making the test consume the corresponding amount of disk.
+ (files / "unselected-large.bin").touch()
+ os.truncate(files / "unselected-large.bin", 256 * 1024 * 1024)
+
+ with private_evaluator_skill_snapshot(target, task_source="evals_json") as snapshot:
+ projected_files = snapshot / "evals" / "files"
+ assert (projected_files / "selected.txt").read_text(encoding="utf-8") == "SELECTED-FIXTURE\n"
+ assert not (projected_files / "hidden.txt").exists()
+ assert not os.path.lexists(projected_files / "unselected-unsafe")
+ assert not (projected_files / "unselected-large.bin").exists()
+
+
[email protected]("kind", ["symlink", "hardlink", "fifo"])
+def test_selective_evaluator_snapshot_fails_closed_for_referenced_unsafe_fixture(
+ tmp_path: Path,
+ kind: str,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ files = target / "evals" / "files"
+ selected = files / "selected.txt"
+ selected.unlink()
+ outside = tmp_path / "outside-selected.txt"
+ outside.write_text("must never be copied\n", encoding="utf-8")
+ try:
+ if kind == "symlink":
+ selected.symlink_to(outside)
+ elif kind == "hardlink":
+ selected.hardlink_to(outside)
+ elif hasattr(os, "mkfifo"):
+ os.mkfifo(selected)
+ else:
+ pytest.skip("FIFO creation is unavailable")
+ except OSError as exc:
+ pytest.skip(f"{kind} creation is unavailable: {exc}")
+
+ with (
+ pytest.raises((FileNotFoundError, ValueError), match=r"outside evals|symlink|hard.?link|regular file"),
+ private_evaluator_skill_snapshot(target, task_source="evals_json"),
+ ):
+ pass
+
+
+def test_selective_evaluator_snapshot_preserves_legacy_implicit_files_corpus(tmp_path: Path) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ (target / "evals" / "evals.json").write_text(
+ json.dumps([{"id": "case-001", "question": "Use the shared fixtures."}]),
+ encoding="utf-8",
+ )
+
+ with private_evaluator_skill_snapshot(target, task_source="evals_json") as snapshot:
+ projected_files = snapshot / "evals" / "files"
+ assert (projected_files / "selected.txt").is_file()
+ assert (projected_files / "hidden.txt").is_file()
+
+
+def test_native_snapshot_ignores_generated_only_entries_when_selecting_fixtures(tmp_path: Path) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ evals = target / "evals"
+ (evals / "evals.json").write_text(
+ json.dumps(
+ [
+ {
+ "id": "case-001",
+ "question": "Use only the selected fixture.",
+ "files": ["evals/files/selected.txt"],
+ },
+ {
+ "id": "generated-only",
+ "question": "This entry has legacy implicit shared files.",
+ },
+ ]
+ ),
+ encoding="utf-8",
+ )
+ unsafe = evals / "files" / "unselected-unsafe"
+ outside = tmp_path / "outside-native-fixtures.txt"
+ outside.write_text("must not be inspected or copied\n", encoding="utf-8")
+ try:
+ unsafe.symlink_to(outside)
+ except OSError as exc:
+ pytest.skip(f"symlink creation is unavailable: {exc}")
+
+ with private_evaluator_skill_snapshot(target, task_source="native_harbor") as snapshot:
+ projected_files = snapshot / "evals" / "files"
+ assert (projected_files / "selected.txt").read_text(encoding="utf-8") == "SELECTED-FIXTURE\n"
+ assert not (projected_files / "hidden.txt").exists()
+ assert not os.path.lexists(projected_files / "unselected-unsafe")
+
+ task = stage_native_harbor_tasks(target, tmp_path / "native-selected-fixtures", with_skill=False)[0]
+ staged_input = task / "environment" / "input"
+ assert (staged_input / "selected.txt").read_text(encoding="utf-8") == "SELECTED-FIXTURE\n"
+ assert not (staged_input / "hidden.txt").exists()
+ assert not os.path.lexists(staged_input / "unselected-unsafe")
+
+
+def test_evidence_snapshot_binds_complete_files_corpus_for_explicit_entries(tmp_path: Path) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+
+ with private_evaluator_skill_snapshot(
+ target,
+ task_source="evals_json",
+ bind_full_evidence_sources=True,
+ ) as snapshot:
+ projected_files = snapshot / "evals" / "files"
+ assert (projected_files / "selected.txt").is_file()
+ assert (projected_files / "hidden.txt").is_file()
+
+
+def test_selective_snapshot_ignores_shadowed_grader_but_evidence_binds_it(tmp_path: Path) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ shadowed = target / "evals" / "tests" / "grader.py"
+ shadowed.parent.mkdir()
+ outside = tmp_path / "outside-shadowed-grader.py"
+ outside.write_text("raise RuntimeError('must not run')\n", encoding="utf-8")
+ try:
+ shadowed.symlink_to(outside)
+ except OSError as exc:
+ pytest.skip(f"symlink creation is unavailable: {exc}")
+
+ with private_evaluator_skill_snapshot(target, task_source="evals_json") as snapshot:
+ assert (snapshot / "evals" / "grader.py").is_file()
+ assert not os.path.lexists(snapshot / "evals" / "tests" / "grader.py")
+
+ with (
+ pytest.raises(ValueError, match="symlink"),
+ private_evaluator_skill_snapshot(
+ target,
+ task_source="evals_json",
+ bind_full_evidence_sources=True,
+ ),
+ ):
+ pass
+
+
+def test_evidence_snapshot_rejects_unselected_unsafe_file_it_must_bind(tmp_path: Path) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ hidden = target / "evals" / "files" / "hidden.txt"
+ hidden.unlink()
+ outside = tmp_path / "evidence-hardlink.txt"
+ outside.write_text("bound evidence\n", encoding="utf-8")
+ try:
+ hidden.hardlink_to(outside)
+ except OSError as exc:
+ pytest.skip(f"hardlinks unavailable: {exc}")
+
+ with (
+ pytest.raises(ValueError, match=r"hard.?link"),
+ private_evaluator_skill_snapshot(
+ target,
+ task_source="evals_json",
+ bind_full_evidence_sources=True,
+ ),
+ ):
+ pass
+
+
[email protected]("kind", ["symlink", "hardlink", "fifo"])
+def test_selective_evaluator_snapshot_ignores_unsafe_unconsumed_evals_subtree(
+ tmp_path: Path,
+ kind: str,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ archive = target / "evals" / "historical-archive"
+ archive.mkdir()
+ unsafe = archive / "unsafe"
+ outside = tmp_path / "outside-archive.txt"
+ outside.write_text("unconsumed\n", encoding="utf-8")
+ try:
+ if kind == "symlink":
+ unsafe.symlink_to(outside)
+ elif kind == "hardlink":
+ unsafe.hardlink_to(outside)
+ elif hasattr(os, "mkfifo"):
+ os.mkfifo(unsafe)
+ else:
+ pytest.skip("FIFO creation is unavailable")
+ except OSError as exc:
+ pytest.skip(f"{kind} creation is unavailable: {exc}")
+
+ with private_evaluator_skill_snapshot(target, task_source="evals_json") as snapshot:
+ assert not (snapshot / "evals" / "historical-archive").exists()
+
+
+def test_selective_evaluator_snapshot_rejects_fixture_selection_change_during_copy(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ dataset = target / "evals" / "evals.json"
+ original_copytree = adapter_module.copytree_secure
+ mutated = False
+
+ def mutate_selection_then_copy(source: Path, destination: Path, *args: object, **kwargs: object) -> None:
+ nonlocal mutated
+ if Path(source) == target / "evals" and not mutated:
+ dataset.write_text(
+ json.dumps([{"id": "case-001", "question": "Now use every shared fixture."}]),
+ encoding="utf-8",
+ )
+ mutated = True
+ original_copytree(source, destination, *args, **kwargs)
+
+ monkeypatch.setattr(adapter_module, "copytree_secure", mutate_selection_then_copy)
+
+ with (
+ pytest.raises(ValueError, match="selection changed"),
+ private_evaluator_skill_snapshot(target, task_source="evals_json"),
+ ):
+ pass
+
+ assert mutated
+
+
+def test_in_repo_temp_evaluator_snapshot_is_excluded_from_copy_repo(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ repo, target, _, _ = _write_projection_fixture(tmp_path)
+ in_repo_temp = repo / "in-repo-temp"
+ in_repo_temp.mkdir()
+ monkeypatch.setattr(tempfile, "tempdir", str(in_repo_temp))
+
+ task = generate_harbor_tasks(
+ target,
+ tmp_path / "in-repo-temp-output",
+ with_skill=True,
+ copy_repo=True,
+ )[0]
+
+ staged_repo = task / "environment" / "repo"
+ assert staged_repo.is_dir()
+ assert not list(staged_repo.rglob("evals.json"))
+ readable = "\n".join(
+ path.read_text(encoding="utf-8", errors="ignore") for path in staged_repo.rglob("*") if path.is_file()
+ )
+ assert "GROUND-TRUTH-SECRET" not in readable
+
+
+def test_temp_snapshot_inside_legacy_files_does_not_recurse_into_itself(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ (target / "evals" / "evals.json").write_text(
+ json.dumps([{"id": "case-001", "question": "Use every shared fixture."}]),
+ encoding="utf-8",
+ )
+ nested_temp = target / "evals" / "files" / "nested-temp-root"
+ nested_temp.mkdir()
+ monkeypatch.setattr(tempfile, "tempdir", str(nested_temp))
+
+ task = generate_harbor_tasks(target, tmp_path / "nested-temp-output", with_skill=False)[0]
+
+ staged_input = task / "environment" / "input"
+ assert (staged_input / "selected.txt").is_file()
+ assert (staged_input / "hidden.txt").is_file()
+ assert not list(staged_input.rglob("evals.json"))
+
+
+def test_native_projection_does_not_enumerate_symlinked_harbor_root(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ outside_skill = tmp_path / "outside-skill"
+ _write_minimal_native_task(outside_skill)
+ outside = outside_skill / "evals" / "harbor"
+ native_root = target / "evals" / "harbor"
+ try:
+ native_root.symlink_to(outside, target_is_directory=True)
+ except OSError as exc:
+ pytest.skip(f"directory symlink creation is unavailable: {exc}")
+
+ original_iterdir = Path.iterdir
+ enumerated_source = False
+
+ def record_iterdir(path: Path):
+ nonlocal enumerated_source
+ if path.absolute() == native_root.absolute():
+ enumerated_source = True
+ return original_iterdir(path)
+
+ monkeypatch.setattr(Path, "iterdir", record_iterdir)
+
+ with pytest.raises(ValueError, match="symlink"):
+ stage_native_harbor_tasks(target, tmp_path / "symlinked-native-root")
+
+ assert not enumerated_source
+
+
+def test_native_projection_does_not_read_task_directory_on_another_device(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ native_root = target / "evals" / "harbor"
+ task_dir = native_root / "case-001"
+ original_lstat = Path.lstat
+ original_read = adapter_module._read_regular_evals_file
+ read_task_config = False
+
+ def cross_device_lstat(path: Path):
+ metadata = original_lstat(path)
+ if path.absolute() != task_dir.absolute():
+ return metadata
+ fields = list(metadata)
+ fields[2] = metadata.st_dev + 1
+ return os.stat_result(fields)
+
+ def record_read(path: Path, **kwargs: object) -> bytes:
+ nonlocal read_task_config
+ if path.absolute() == (task_dir / "task.toml").absolute():
+ read_task_config = True
+ return original_read(path, **kwargs)
+
+ monkeypatch.setattr(Path, "lstat", cross_device_lstat)
+ monkeypatch.setattr(adapter_module, "_read_regular_evals_file", record_read)
+
+ assert adapter_module._native_projection_entry_ids(native_root) == ()
+ assert not read_task_config
diff --git a/tests/test_harbor_secure_copy.py b/tests/test_harbor_secure_copy.py
index 3242b871..ba25ba57 100644
--- a/tests/test_harbor_secure_copy.py
+++ b/tests/test_harbor_secure_copy.py
@@ -654,6 +654,84 @@ def test_checked_non_posix_fallback_copies_tree_and_file(
assert stat.S_IMODE(file_destination.stat().st_mode) == 0o640
+def test_checked_non_posix_tree_fallback_without_fchmod_preserves_file_mode(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ module = importlib.import_module("skillevaluator.tier3.harbor.secure_copy")
+ monkeypatch.setattr(module, "_DESCRIPTOR_BACKEND", False)
+ monkeypatch.setattr(module, "_ATOMIC_RENAME", None)
+ monkeypatch.delattr(module.os, "fchmod")
+ source = tmp_path / "source"
+ source.mkdir()
+ source_file = source / "safe.txt"
+ source_file.write_text("safe", encoding="utf-8")
+ source_file.chmod(0o640)
+ destination = tmp_path / "destination"
+
+ module.copytree_secure(source, destination, allowed_root=tmp_path)
+
+ staged_file = destination / "safe.txt"
+ assert staged_file.read_text(encoding="utf-8") == "safe"
+ assert stat.S_IMODE(staged_file.stat().st_mode) == 0o640
+
+
+def test_checked_non_posix_file_fallback_without_fchmod_preserves_mode(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ module = importlib.import_module("skillevaluator.tier3.harbor.secure_copy")
+ monkeypatch.setattr(module, "_DESCRIPTOR_BACKEND", False)
+ monkeypatch.setattr(module, "_ATOMIC_RENAME", None)
+ monkeypatch.delattr(module.os, "fchmod")
+ source = tmp_path / "source.txt"
+ source.write_text("safe", encoding="utf-8")
+ source.chmod(0o640)
+ destination = tmp_path / "destination.txt"
+
+ module.copy_file_secure(source, destination, allowed_root=tmp_path)
+
+ assert destination.read_text(encoding="utf-8") == "safe"
+ assert stat.S_IMODE(destination.stat().st_mode) == 0o640
+
+
+def test_checked_windows_fallback_verifies_portable_chmod_semantics(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ module = importlib.import_module("skillevaluator.tier3.harbor.secure_copy")
+ source = tmp_path / "source"
+ source.mkdir()
+ source_file = source / "writable.txt"
+ source_file.write_text("tree", encoding="utf-8")
+ source_file.chmod(0o640)
+ file_source = tmp_path / "read-only.txt"
+ file_source.write_text("file", encoding="utf-8")
+ file_source.chmod(0o440)
+ original_chmod = Path.chmod
+
+ def windows_chmod(path: Path, mode: int, *args: object, **kwargs: object) -> None:
+ executable = 0o111 if path.is_dir() else 0
+ portable_mode = (0o666 | executable) if mode & stat.S_IWRITE else (0o444 | executable)
+ original_chmod(path, portable_mode, *args, **kwargs)
+
+ monkeypatch.setattr(module, "_DESCRIPTOR_BACKEND", False)
+ monkeypatch.setattr(module, "_ATOMIC_RENAME", None)
+ monkeypatch.setattr(module, "_WINDOWS_CHMOD_SEMANTICS", True)
+ monkeypatch.delattr(module.os, "fchmod")
+ monkeypatch.setattr(module.Path, "chmod", windows_chmod)
+ tree_destination = tmp_path / "tree-destination"
+ file_destination = tmp_path / "file-destination.txt"
+
+ module.copytree_secure(source, tree_destination, allowed_root=tmp_path)
+ module.copy_file_secure(file_source, file_destination, allowed_root=tmp_path)
+
+ assert (tree_destination / "writable.txt").read_text(encoding="utf-8") == "tree"
+ assert (tree_destination / "writable.txt").stat().st_mode & stat.S_IWRITE
+ assert file_destination.read_text(encoding="utf-8") == "file"
+ assert not (file_destination.stat().st_mode & stat.S_IWRITE)
+
+
def test_checked_non_posix_fallback_still_rejects_symlinks(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/test_results_location.py b/tests/test_results_location.py
index 464fc828..5820596c 100644
--- a/tests/test_results_location.py
+++ b/tests/test_results_location.py
@@ -19,6 +19,7 @@ from skillevaluator.tier3.results_location import external_results_root, resolve
def _write_completed_run(root: Path, run_id: str) -> Path:
run_dir = root / run_id
run_dir.mkdir(parents=True)
+ (run_dir / "run_config.json").write_text("{}", encoding="utf-8")
(run_dir / "result.json").write_text(
json.dumps({"run_id": run_id, "agents": {}}),
encoding="utf-8",
diff --git a/tests/test_results_location_legacy_compatibility.py b/tests/test_results_location_legacy_compatibility.py
new file mode 100644
index 00000000..7b9a563f
--- /dev/null
+++ b/tests/test_results_location_legacy_compatibility.py
@@ -0,0 +1,734 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Compatibility and race contracts for pre-result Tier 3 run directories."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+import pytest
+
+from skillevaluator.tier3 import results_location
+from skillevaluator.tier3.output_provenance import GENERATED_OUTPUT_MARKER, mark_generated_output_root
+
+
+def _run(root: Path, name: str = "20260709_120000_1_aaaaaa") -> Path:
+ candidate = root / name
+ candidate.mkdir(parents=True)
+ return candidate
+
+
+def _write_legacy_run_config(run: Path, *agents: str) -> None:
+ (run / "run_config.json").write_text(
+ json.dumps(
+ {
+ "harbor": {"n_attempts": {"value": 1, "source": "ACES default"}},
+ "task_source": "evals_json",
+ "agents": {
+ agent: {
+ "agent": agent,
+ "model": "test-model",
+ "source": "test default",
+ "occurrence": "1",
+ }
+ for agent in agents
+ },
+ }
+ ),
+ encoding="utf-8",
+ )
+
+
+def _pass_at_k_payload() -> dict[str, object]:
+ return {
+ "k": 1,
+ "pass_threshold": 0.5,
+ "stop_on_pass": False,
+ "passed_cases": 1,
+ "failed_cases": 0,
+ "total_cases": 1,
+ "rate": 1.0,
+ "attempts_used": 1,
+ "max_attempts_possible": 1,
+ "avg_attempts_used": 1.0,
+ "extra_cases": [],
+ "cases": {
+ "case": {
+ "passed": True,
+ "first_pass_attempt": 1,
+ "attempts_used": 1,
+ "attempts_skipped": 0,
+ "attempts_missing": 0,
+ "best_score": 0.8,
+ "attempts": [{"attempt": 1, "trial": "case__1", "score": 0.8, "passed": True}],
+ }
+ },
+ }
+
+
+def _successful_summary_payload(agent: str = "opencode") -> dict[str, object]:
+ return {
+ "agent": agent,
+ "model": "test-model",
+ "model_source": "test default",
+ "scores": {"security": 0.8},
+ "custom_scores": {},
+ "metric_set": "aces-default-v2",
+ "metrics": ["security"],
+ "dimensions": {"safety": {"score": 0.8, "sources": {"security": 1.0}}},
+ "num_trials": 1,
+ "pass_at_k": _pass_at_k_payload(),
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ "job_failure": "",
+ "trial_failures": [],
+ }
+
+
+def _write_successful_summary(run: Path, *, agent: str = "opencode") -> Path:
+ summary = run / agent / "with-skill" / "summary.json"
+ summary.parent.mkdir(parents=True)
+ summary.write_text(json.dumps(_successful_summary_payload(agent)), encoding="utf-8")
+ return summary
+
+
+def _write_current_run(root: Path, name: str = "20260709_120000_1_aaaaaa") -> Path:
+ run = _run(root, name)
+ (run / "run_config.json").write_text("{}", encoding="utf-8")
+ (run / "result.json").write_text(json.dumps({"run_id": name, "agents": {}}), encoding="utf-8")
+ return run
+
+
+def _is_completed(run: Path) -> bool:
+ return results_location.run_directory_sort_key(run, require_completed_result=True) is not None
+
+
+def test_rejects_current_run_with_list_valued_run_config(tmp_path: Path) -> None:
+ run = _write_current_run(tmp_path)
+ (run / "run_config.json").write_text("[]", encoding="utf-8")
+
+ assert not _is_completed(run)
+
+
+def test_accepts_pre_result_run_with_valid_summary(tmp_path: Path) -> None:
+ run = _run(tmp_path)
+ _write_legacy_run_config(run, "opencode")
+ _write_successful_summary(run)
+
+ assert _is_completed(run)
+
+
+def test_accepts_summary_from_before_truthful_status_fields(tmp_path: Path) -> None:
+ run = _run(tmp_path)
+ _write_legacy_run_config(run, "opencode")
+ summary = run / "opencode" / "with-skill" / "summary.json"
+ summary.parent.mkdir(parents=True)
+ summary.write_text(
+ json.dumps(
+ {
+ "agent": "opencode",
+ "model": "test-model",
+ "model_source": "test default",
+ "scores": {"security": 0.8},
+ "custom_scores": {},
+ "metric_set": "aces-default-v2",
+ "metrics": ["security"],
+ "dimensions": {"safety": {"score": 0.8, "sources": {"security": 1.0}}},
+ "num_trials": 1,
+ "pass_at_k": _pass_at_k_payload(),
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ assert _is_completed(run)
+
+
+ "run_config_payload",
+ [
+ "not-json",
+ "[]",
+ "null",
+ "1",
+ "{}",
+ '{"agents": {}}',
+ json.dumps(
+ {
+ "harbor": {"n_attempts": {"value": 1, "source": "ACES default"}},
+ "task_source": "generated",
+ "agents": {
+ "opencode": {
+ "agent": "opencode",
+ "model": "test-model",
+ "source": "test default",
+ "occurrence": "1",
+ }
+ },
+ }
+ ),
+ json.dumps(
+ {
+ "harbor": {"n_attempts": {"value": 1, "source": "ACES default"}},
+ "task_source": "evals_json",
+ "agents": {"opencode": {}},
+ }
+ ),
+ ],
+)
+def test_rejects_legacy_run_with_invalid_run_config(tmp_path: Path, run_config_payload: str) -> None:
+ run = _run(tmp_path)
+ (run / "run_config.json").write_text(run_config_payload, encoding="utf-8")
+ _write_successful_summary(run)
+
+ assert not _is_completed(run)
+
+
+def test_requires_summary_for_every_configured_agent(tmp_path: Path) -> None:
+ run = _run(tmp_path)
+ _write_legacy_run_config(run, "opencode", "claude")
+ _write_successful_summary(run, agent="opencode")
+
+ assert not _is_completed(run)
+
+
+def test_rejects_malformed_present_baseline_summary(tmp_path: Path) -> None:
+ run = _run(tmp_path)
+ _write_legacy_run_config(run, "opencode")
+ _write_successful_summary(run)
+ baseline = run / "opencode" / "without-skill" / "summary.json"
+ baseline.parent.mkdir()
+ baseline.write_text("not-json", encoding="utf-8")
+
+ assert not _is_completed(run)
+
+
+def test_rejects_ambiguous_flat_and_nested_summaries(tmp_path: Path) -> None:
+ run = _run(tmp_path)
+ _write_legacy_run_config(run, "opencode")
+ _write_successful_summary(run)
+ (run / "opencode" / "summary.json").write_text(
+ json.dumps(_successful_summary_payload()),
+ encoding="utf-8",
+ )
+
+ assert not _is_completed(run)
+
+
+def test_accepts_authentic_pre_status_custom_only_summary(tmp_path: Path) -> None:
+ run = _run(tmp_path)
+ _write_legacy_run_config(run, "opencode")
+ payload = _successful_summary_payload()
+ for field in (
+ "execution_status",
+ "execution_errors",
+ "expected_attempts",
+ "scored_attempts",
+ "job_failure",
+ "trial_failures",
+ ):
+ payload.pop(field)
+ payload.update(
+ {
+ "scores": {},
+ "custom_scores": {},
+ "metric_set": "custom-only",
+ "metrics": [],
+ "dimensions": {},
+ }
+ )
+ summary = run / "opencode" / "with-skill" / "summary.json"
+ summary.parent.mkdir(parents=True)
+ summary.write_text(json.dumps(payload), encoding="utf-8")
+
+ assert _is_completed(run)
+
+
+def test_accepts_truthful_status_era_failed_summary(tmp_path: Path) -> None:
+ run = _run(tmp_path)
+ _write_legacy_run_config(run, "opencode")
+ payload = _successful_summary_payload()
+ payload.update(
+ {
+ "scores": {},
+ "dimensions": {},
+ "num_trials": 0,
+ "pass_at_k": {
+ "k": 1,
+ "pass_threshold": 0.5,
+ "stop_on_pass": False,
+ "passed_cases": 0,
+ "failed_cases": 1,
+ "total_cases": 1,
+ "rate": 0.0,
+ "attempts_used": 0,
+ "max_attempts_possible": 1,
+ "avg_attempts_used": 0.0,
+ "extra_cases": [],
+ "cases": {
+ "case": {
+ "passed": False,
+ "first_pass_attempt": None,
+ "attempts_used": 0,
+ "attempts_skipped": 0,
+ "attempts_missing": 1,
+ "best_score": None,
+ "attempts": [],
+ }
+ },
+ },
+ "execution_status": "failed",
+ "execution_errors": ["Harbor job directory was not created"],
+ "expected_attempts": 1,
+ "scored_attempts": 0,
+ "job_failure": "Harbor job directory was not created",
+ "trial_failures": [],
+ }
+ )
+ summary = run / "opencode" / "with-skill" / "summary.json"
+ summary.parent.mkdir(parents=True)
+ summary.write_text(json.dumps(payload), encoding="utf-8")
+
+ assert _is_completed(run)
+
+
+def test_rejects_status_summary_with_incoherent_scored_attempts(tmp_path: Path) -> None:
+ run = _run(tmp_path)
+ _write_legacy_run_config(run, "opencode")
+ payload = _successful_summary_payload()
+ payload.update(
+ {
+ "execution_status": "failed",
+ "execution_errors": ["Harbor reported a failure"],
+ "scored_attempts": 0,
+ "job_failure": "Harbor reported a failure",
+ }
+ )
+ summary = _write_successful_summary(run)
+ summary.write_text(json.dumps(payload), encoding="utf-8")
+
+ assert not _is_completed(run)
+
+
+def test_accepts_truthful_failed_excess_attempt_coverage(tmp_path: Path) -> None:
+ run = _run(tmp_path)
+ _write_legacy_run_config(run, "opencode")
+ payload = _successful_summary_payload()
+ pass_at_k = _pass_at_k_payload()
+ attempts = [
+ {"attempt": 1, "trial": "case__attempt1", "score": 0.8, "passed": True},
+ {"attempt": 2, "trial": "case__attempt2", "score": 0.7, "passed": True},
+ ]
+ pass_at_k.update({"attempts_used": 2, "avg_attempts_used": 2.0})
+ pass_at_k["cases"] = {
+ "case": {
+ "passed": True,
+ "first_pass_attempt": 1,
+ "attempts_used": 2,
+ "attempts_skipped": 0,
+ "attempts_missing": 0,
+ "best_score": 0.8,
+ "attempts": attempts,
+ }
+ }
+ payload.update(
+ {
+ "num_trials": 2,
+ "pass_at_k": pass_at_k,
+ "execution_status": "failed",
+ "execution_errors": ["Excess scored attempts for cases: case", "Scored attempt coverage is 2/1"],
+ "expected_attempts": 1,
+ "scored_attempts": 2,
+ "job_failure": "",
+ }
+ )
+ summary = _write_successful_summary(run)
+ summary.write_text(json.dumps(payload), encoding="utf-8")
+
+ assert _is_completed(run)
+
+
+def test_rejects_legacy_summary_without_authentic_pass_at_k(tmp_path: Path) -> None:
+ run = _run(tmp_path)
+ _write_legacy_run_config(run, "opencode")
+ payload = _successful_summary_payload()
+ payload["pass_at_k"] = {}
+ summary = run / "opencode" / "with-skill" / "summary.json"
+ summary.parent.mkdir(parents=True)
+ summary.write_text(json.dumps(payload), encoding="utf-8")
+
+ assert not _is_completed(run)
+
+
+ ("field", "value"),
+ [
+ ("passed_cases", 0),
+ ("attempts_used", 0),
+ ("rate", 0.0),
+ ],
+)
+def test_rejects_incoherent_legacy_pass_at_k_totals(
+ tmp_path: Path,
+ field: str,
+ value: object,
+) -> None:
+ run = _run(tmp_path)
+ _write_legacy_run_config(run, "opencode")
+ payload = _successful_summary_payload()
+ payload["pass_at_k"] = {**_pass_at_k_payload(), field: value}
+ summary = _write_successful_summary(run)
+ summary.write_text(json.dumps(payload), encoding="utf-8")
+
+ assert not _is_completed(run)
+
+
+ ("field", "value"),
+ [
+ ("passed", False),
+ ("first_pass_attempt", None),
+ ("best_score", 0.7),
+ ("attempts_missing", 1),
+ ],
+)
+def test_rejects_incoherent_legacy_pass_at_k_case(
+ tmp_path: Path,
+ field: str,
+ value: object,
+) -> None:
+ run = _run(tmp_path)
+ _write_legacy_run_config(run, "opencode")
+ payload = _successful_summary_payload()
+ pass_at_k = _pass_at_k_payload()
+ pass_at_k["cases"] = {
+ "case": {
+ **pass_at_k["cases"]["case"],
+ field: value,
+ }
+ }
+ payload["pass_at_k"] = pass_at_k
+ summary = _write_successful_summary(run)
+ summary.write_text(json.dumps(payload), encoding="utf-8")
+
+ assert not _is_completed(run)
+
+
+def test_rejects_legacy_summary_for_wrong_agent(tmp_path: Path) -> None:
+ run = _run(tmp_path)
+ _write_legacy_run_config(run, "opencode")
+ summary = _write_successful_summary(run)
+ summary.write_text(json.dumps(_successful_summary_payload("claude")), encoding="utf-8")
+
+ assert not _is_completed(run)
+
+
+def test_rejects_legacy_run_with_malformed_sibling_summary(tmp_path: Path) -> None:
+ run = _run(tmp_path)
+ _write_legacy_run_config(run, "opencode", "claude")
+ _write_successful_summary(run)
+ malformed = run / "claude" / "with-skill" / "summary.json"
+ malformed.parent.mkdir(parents=True)
+ malformed.write_text("not-json", encoding="utf-8")
+
+ assert not _is_completed(run)
+
+
+ "summary_payload",
+ [
+ {**_successful_summary_payload(), "scores": {"security": 0.8, "malformed": "not-a-number"}},
+ {**_successful_summary_payload(), "scores": {"security": 0.8, "non_finite": float("nan")}},
+ {**_successful_summary_payload(), "scores": {"security": 0.8, "overflowing": 10**1000}},
+ {**_successful_summary_payload(), "custom_scores": {"quality": 0.6, "malformed": None}},
+ {key: value for key, value in _successful_summary_payload().items() if key != "scores"},
+ {key: value for key, value in _successful_summary_payload().items() if key != "custom_scores"},
+ {key: value for key, value in _successful_summary_payload().items() if key != "model"},
+ {key: value for key, value in _successful_summary_payload().items() if key != "model_source"},
+ {key: value for key, value in _successful_summary_payload().items() if key != "metric_set"},
+ {key: value for key, value in _successful_summary_payload().items() if key != "metrics"},
+ {key: value for key, value in _successful_summary_payload().items() if key != "dimensions"},
+ {key: value for key, value in _successful_summary_payload().items() if key != "pass_at_k"},
+ {key: value for key, value in _successful_summary_payload().items() if key != "job_failure"},
+ {key: value for key, value in _successful_summary_payload().items() if key != "trial_failures"},
+ {**_successful_summary_payload(), "scores": []},
+ ],
+)
+def test_rejects_legacy_summary_with_incomplete_score_maps(
+ tmp_path: Path,
+ summary_payload: dict[str, object],
+) -> None:
+ run = _run(tmp_path)
+ _write_legacy_run_config(run, "opencode")
+ summary = _write_successful_summary(run)
+ summary.write_text(json.dumps(summary_payload), encoding="utf-8")
+
+ assert not _is_completed(run)
+
+
+ "optional_fields",
+ [
+ {"execution_status": "succeeded"},
+ {"execution_errors": []},
+ {"execution_status": "succeeded", "execution_errors": []},
+ {"expected_attempts": 1, "scored_attempts": 1},
+ {"execution_status": "succeeded", "execution_errors": [], "expected_attempts": 1},
+ ],
+)
+def test_rejects_legacy_summary_with_partial_completion_fields(
+ tmp_path: Path,
+ optional_fields: dict[str, object],
+) -> None:
+ run = _run(tmp_path)
+ _write_legacy_run_config(run, "opencode")
+ payload = _successful_summary_payload()
+ for field in (
+ "execution_status",
+ "execution_errors",
+ "expected_attempts",
+ "scored_attempts",
+ "job_failure",
+ "trial_failures",
+ ):
+ payload.pop(field)
+ payload.update(optional_fields)
+ summary = _write_successful_summary(run)
+ summary.write_text(json.dumps(payload), encoding="utf-8")
+
+ assert not _is_completed(run)
+
+
+ "summary_payload",
+ [
+ "not-json",
+ json.dumps([]),
+ json.dumps(
+ {
+ "agent": "opencode",
+ "scores": {"security": 0.8},
+ "custom_scores": {},
+ "num_trials": 1,
+ "execution_status": "failed",
+ "execution_errors": ["trial failed"],
+ "expected_attempts": 1,
+ "scored_attempts": 0,
+ }
+ ),
+ json.dumps(
+ {
+ "agent": "opencode",
+ "scores": {"security": 0.8},
+ "custom_scores": {},
+ "num_trials": 1,
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 2,
+ "scored_attempts": 1,
+ }
+ ),
+ ],
+)
+def test_rejects_legacy_run_without_valid_completion_summary(tmp_path: Path, summary_payload: str) -> None:
+ run = _run(tmp_path)
+ _write_legacy_run_config(run, "opencode")
+ summary = run / "opencode" / "with-skill" / "summary.json"
+ summary.parent.mkdir(parents=True)
+ summary.write_text(summary_payload, encoding="utf-8")
+
+ assert not _is_completed(run)
+
+
+def test_rejects_authenticated_current_partial_with_legacy_summary(tmp_path: Path) -> None:
+ run = _run(tmp_path)
+ mark_generated_output_root(run)
+ _write_legacy_run_config(run, "opencode")
+ _write_successful_summary(run)
+
+ assert not _is_completed(run)
+
+
+def test_rejects_invalid_current_marker_downgrade_to_legacy(tmp_path: Path) -> None:
+ run = _run(tmp_path)
+ (run / GENERATED_OUTPUT_MARKER).write_text("tampered", encoding="utf-8")
+ _write_legacy_run_config(run, "opencode")
+ _write_successful_summary(run)
+
+ assert not _is_completed(run)
+
+
+def test_rejects_legacy_run_config_replaced_before_open(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ run = _run(tmp_path)
+ run_config = run / "run_config.json"
+ _write_legacy_run_config(run, "opencode")
+ run_config_payload = run_config.read_text(encoding="utf-8")
+ _write_successful_summary(run)
+ real_read_bytes = results_location.SecureRoot.read_bytes
+ replaced = False
+
+ def replace_before_open(
+ secure_root: results_location.SecureRoot,
+ relative_path: Path,
+ max_bytes: int,
+ *,
+ expected: os.stat_result | None = None,
+ ) -> tuple[bytes, os.stat_result]:
+ nonlocal replaced
+ if not replaced and relative_path == Path("run_config.json"):
+ replacement = run_config.with_suffix(".replacement")
+ replacement.write_text(run_config_payload, encoding="utf-8")
+ replacement.replace(run_config)
+ replaced = True
+ return real_read_bytes(secure_root, relative_path, max_bytes, expected=expected)
+
+ monkeypatch.setattr(results_location.SecureRoot, "read_bytes", replace_before_open)
+
+ assert not _is_completed(run)
+
+
+def test_rejects_same_inode_summary_rewrite_with_restored_mtime(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ run = _run(tmp_path)
+ _write_legacy_run_config(run, "opencode")
+ summary = _write_successful_summary(run)
+ summary_metadata = summary.stat()
+ summary_inode = summary_metadata.st_ino
+ original = summary.read_text(encoding="utf-8")
+ rewritten = original.replace("0.8", "0.9")
+ assert len(rewritten) == len(original)
+ real_read = os.read
+ rewritten_once = False
+
+ def rewrite_during_read(fd: int, length: int) -> bytes:
+ nonlocal rewritten_once
+ if not rewritten_once and os.fstat(fd).st_ino == summary_inode:
+ summary.write_text(rewritten, encoding="utf-8")
+ os.utime(summary, ns=(summary_metadata.st_atime_ns, summary_metadata.st_mtime_ns))
+ rewritten_once = True
+ return real_read(fd, length)
+
+ monkeypatch.setattr(results_location.os, "read", rewrite_during_read)
+
+ assert not _is_completed(run)
+
+
+def test_rejects_summary_parent_swapped_to_link_before_read(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ run = _run(tmp_path)
+ _write_legacy_run_config(run, "opencode")
+ summary = _write_successful_summary(run)
+ condition_dir = summary.parent
+ moved = run / "opencode" / "moved-with-skill"
+ real_read_bytes = results_location.SecureRoot.read_bytes
+ swapped = False
+
+ def swap_summary_parent(
+ secure_root: results_location.SecureRoot,
+ relative_path: Path,
+ max_bytes: int,
+ *,
+ expected: os.stat_result | None = None,
+ ) -> tuple[bytes, os.stat_result]:
+ nonlocal swapped
+ if not swapped and relative_path == Path("opencode/with-skill/summary.json"):
+ swapped = True
+ condition_dir.rename(moved)
+ condition_dir.symlink_to(moved, target_is_directory=True)
+ return real_read_bytes(secure_root, relative_path, max_bytes, expected=expected)
+
+ monkeypatch.setattr(results_location.SecureRoot, "read_bytes", swap_summary_parent)
+
+ assert not _is_completed(run)
+
+
+def test_rejects_current_result_replaced_during_read(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ run = _write_current_run(tmp_path)
+ result_path = run / "result.json"
+ result_inode = result_path.stat().st_ino
+ result_payload = result_path.read_text(encoding="utf-8")
+ real_read = os.read
+ replaced = False
+
+ def replace_after_read(fd: int, length: int) -> bytes:
+ nonlocal replaced
+ content = real_read(fd, length)
+ if not replaced and os.fstat(fd).st_ino == result_inode:
+ replacement = result_path.with_suffix(".replacement")
+ replacement.write_text(result_payload, encoding="utf-8")
+ replacement.replace(result_path)
+ replaced = True
+ return content
+
+ monkeypatch.setattr(results_location.os, "read", replace_after_read)
+
+ assert not _is_completed(run)
+
+
+def test_rejects_run_config_changed_during_result_read(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ run = _write_current_run(tmp_path)
+ result_path = run / "result.json"
+ result_inode = result_path.stat().st_ino
+ run_config = run / "run_config.json"
+ config_metadata = run_config.stat()
+ real_read = os.read
+ changed = False
+
+ def change_config_after_result_read(fd: int, length: int) -> bytes:
+ nonlocal changed
+ content = real_read(fd, length)
+ if not changed and os.fstat(fd).st_ino == result_inode:
+ run_config.write_text("[]", encoding="utf-8")
+ os.utime(run_config, ns=(config_metadata.st_atime_ns, config_metadata.st_mtime_ns))
+ changed = True
+ return content
+
+ monkeypatch.setattr(results_location.os, "read", change_config_after_result_read)
+
+ assert not _is_completed(run)
+
+
+def test_rejects_candidate_tree_swap_between_artifact_reads(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ run = _write_current_run(tmp_path)
+ moved = tmp_path / "moved-run"
+ real_read_bytes = results_location.SecureRoot.read_bytes
+ swapped = False
+
+ def swap_candidate(
+ secure_root: results_location.SecureRoot,
+ relative_path: Path,
+ max_bytes: int,
+ *,
+ expected: os.stat_result | None = None,
+ ) -> tuple[bytes, os.stat_result]:
+ nonlocal swapped
+ if not swapped and relative_path == Path("result.json"):
+ swapped = True
+ run.rename(moved)
+ _write_current_run(tmp_path, run.name)
+ return real_read_bytes(secure_root, relative_path, max_bytes, expected=expected)
+
+ monkeypatch.setattr(results_location.SecureRoot, "read_bytes", swap_candidate)
+
+ assert not _is_completed(run)
diff --git a/tests/test_tier3_compare.py b/tests/test_tier3_compare.py
index 7085e5db..d9c290ee 100644
--- a/tests/test_tier3_compare.py
+++ b/tests/test_tier3_compare.py
@@ -15,12 +15,84 @@ from skillevaluator.tier3.commands import compare_results
def _complete_current_run(run_dir: Path) -> None:
+ (run_dir / "run_config.json").write_text("{}", encoding="utf-8")
(run_dir / "result.json").write_text(
json.dumps({"run_id": run_dir.name, "agents": {}}),
encoding="utf-8",
)
+def _write_authentic_pre_status_run(root: Path, run_id: str, *, score: float = 0.8) -> Path:
+ run_dir = root / run_id
+ summary_dir = run_dir / "opencode" / "with-skill"
+ summary_dir.mkdir(parents=True)
+ (run_dir / "run_config.json").write_text(
+ json.dumps(
+ {
+ "harbor": {"n_attempts": {"value": 1, "source": "ACES default"}},
+ "task_source": "evals_json",
+ "agents": {
+ "opencode": {
+ "agent": "opencode",
+ "model": "test-model",
+ "source": "test default",
+ "occurrence": "1",
+ }
+ },
+ }
+ ),
+ encoding="utf-8",
+ )
+ (summary_dir / "summary.json").write_text(
+ json.dumps(
+ {
+ "agent": "opencode",
+ "model": "test-model",
+ "model_source": "test default",
+ "scores": {"security": score},
+ "custom_scores": {},
+ "metric_set": "aces-default-v2",
+ "metrics": ["security"],
+ "dimensions": {"safety": {"score": score, "sources": {"security": 1.0}}},
+ "num_trials": 1,
+ "pass_at_k": {
+ "k": 1,
+ "pass_threshold": 0.5,
+ "stop_on_pass": False,
+ "passed_cases": 1,
+ "failed_cases": 0,
+ "total_cases": 1,
+ "rate": 1.0,
+ "attempts_used": 1,
+ "max_attempts_possible": 1,
+ "avg_attempts_used": 1.0,
+ "extra_cases": [],
+ "cases": {
+ "case": {
+ "passed": True,
+ "first_pass_attempt": 1,
+ "attempts_used": 1,
+ "attempts_skipped": 0,
+ "attempts_missing": 0,
+ "best_score": score,
+ "attempts": [
+ {
+ "attempt": 1,
+ "trial": "case__attempt1",
+ "score": score,
+ "passed": True,
+ }
+ ],
+ }
+ },
+ },
+ }
+ ),
+ encoding="utf-8",
+ )
+ return run_dir
+
+
def test_compare_rejects_partial_scores_from_failed_summary(tmp_path: Path) -> None:
skill_path = tmp_path / "demo"
skill_path.mkdir()
@@ -124,6 +196,7 @@ def test_compare_same_timestamp_unique_runs_uses_result_completion_time(
encoding="utf-8",
)
result_path = run_dir / "result.json"
+ (run_dir / "run_config.json").write_text("{}", encoding="utf-8")
result_path.write_text(json.dumps({"run_id": run_id, "agents": {}}), encoding="utf-8")
os.utime(result_path, ns=(completed_ns, completed_ns))
@@ -187,6 +260,21 @@ def test_compare_retains_non_timestamp_legacy_summary_directory(
assert legacy.name in capsys.readouterr().out
+def test_compare_consumes_authenticated_timestamped_pre_status_run(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ skill_path = tmp_path / "demo"
+ skill_path.mkdir()
+ run_id = "20260709_010000_111_aaaaaaaaaaaa"
+ _write_authentic_pre_status_run(tmp_path / "results" / skill_path.name, run_id)
+
+ assert compare_results(skill_path, results_dir=tmp_path / "results") == 0
+ output = capsys.readouterr().out
+ assert run_id in output
+ assert "opencode" in output
+
+
@pytest.mark.parametrize("primary_kind", ["empty", "partial", "non-object"])
def test_compare_falls_back_from_unusable_primary_root_to_valid_legacy_root(
tmp_path: Path,
diff --git a/tests/test_tier3_progress.py b/tests/test_tier3_progress.py
index a13f71a6..fcfec4cf 100644
--- a/tests/test_tier3_progress.py
+++ b/tests/test_tier3_progress.py
@@ -689,6 +689,84 @@ def test_default_run_cleans_transient_harbor_artifacts(
assert persisted["run_config"]["harbor"]["jobs_retained"] is False
+def test_runner_returns_error_when_private_evaluator_snapshot_cannot_be_created(tmp_path: Path) -> None:
+ from skillevaluator.tier3.harbor import runner
+
+ skill = tmp_path / "missing-evals-skill"
+ skill.mkdir()
+ (skill / "SKILL.md").write_text("# Missing evals\n", encoding="utf-8")
+
+ result = runner.run_harbor_eval(skill, ["codex"])
+
+ assert result["error"] == [f"No evaluator source directory found at {skill / 'evals'}"]
+
+
[email protected]("kind", ["symlink", "hardlink"])
+def test_runner_returns_error_for_unsafe_evals_config(tmp_path: Path, kind: str) -> None:
+ from skillevaluator.tier3.harbor import runner
+
+ skill = tmp_path / "unsafe-config-skill"
+ (skill / "evals").mkdir(parents=True)
+ (skill / "evals" / "evals.json").write_text("[]\n", encoding="utf-8")
+ outside = tmp_path / "outside-config.yml"
+ outside.write_text("schema_version: 1\n", encoding="utf-8")
+ config = skill / "evals" / "config.yml"
+ try:
+ if kind == "symlink":
+ config.symlink_to(outside)
+ else:
+ config.hardlink_to(outside)
+ except OSError as exc:
+ pytest.skip(f"{kind} creation is unavailable: {exc}")
+
+ result = runner.run_harbor_eval(skill, ["codex"])
+
+ assert result["error"] == [f"Eval configuration must be a regular non-linked file: {config}"]
+
+
+def test_all_agent_and_baseline_arms_share_one_private_evaluator_snapshot(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ snapshots: list[Path] = []
+ observed_questions: list[str] = []
+ skill_ref: list[Path] = []
+
+ def emit_tasks(_skill: Path, output: Path, **kwargs: Any) -> list[Path]:
+ snapshot = Path(kwargs["evaluator_skill_path"])
+ snapshots.append(snapshot)
+ entries = json.loads((snapshot / "evals" / "evals.json").read_text(encoding="utf-8"))
+ observed_questions.append(entries[0]["question"])
+ if len(snapshots) == 1:
+ (skill_ref[0] / "evals" / "evals.json").write_text(
+ json.dumps([{"id": "case-1", "question": "Mutated after snapshot."}]),
+ encoding="utf-8",
+ )
+ task = output / "case-1"
+ task.mkdir(parents=True)
+ return [task]
+
+ runner, skill = _stub_runner(monkeypatch, tmp_path, task_emitter=emit_tasks)
+ skill_ref.append(skill)
+ (skill / "evals" / "evals.json").write_text(
+ json.dumps([{"id": "case-1", "question": "Use the original fixture."}]),
+ encoding="utf-8",
+ )
+
+ result = runner.run_harbor_eval(
+ skill,
+ ["codex", "opencode"],
+ output_dir=tmp_path / "results",
+ agent_runtime_preflight=False,
+ )
+
+ assert "error" not in result
+ assert len(snapshots) == 4
+ assert len(set(snapshots)) == 1
+ assert observed_questions == ["Use the original fixture."] * 4
+ assert not snapshots[0].exists()
+
+
def test_runner_rejects_preexisting_run_symlink_before_child_creation(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
@@ -864,10 +942,17 @@ def test_cleanup_failure_degrades_terminal_progress_after_retention_finalizes(
reporter = _RecordingReporter()
from skillevaluator.tier3.harbor import artifact_retention
+ real_rmtree = artifact_retention.shutil.rmtree
+
+ def fail_harbor_artifact_cleanup(path: str | Path, *args: Any, **kwargs: Any) -> None:
+ if Path(path).name.startswith("_harbor-"):
+ raise OSError(f"busy: {path}")
+ real_rmtree(path, *args, **kwargs)
+
monkeypatch.setattr(
artifact_retention.shutil,
"rmtree",
- lambda path: (_ for _ in ()).throw(OSError(f"busy: {path}")),
+ fail_harbor_artifact_cleanup,
)
result = runner.run_harbor_eval(
@@ -1055,6 +1140,9 @@ def test_runner_reports_known_plan_without_claiming_failed_preflight_ready(
from skillevaluator.tier3.harbor import runner
reporter = _RecordingReporter()
+ skill = tmp_path / "demo"
+ (skill / "evals").mkdir(parents=True)
+ (skill / "evals" / "evals.json").write_text("[]\n", encoding="utf-8")
monkeypatch.setattr(
runner,
"resolve_llm_provider",
@@ -1070,7 +1158,7 @@ def test_runner_reports_known_plan_without_claiming_failed_preflight_ready(
)
monkeypatch.setattr(runner, "_check_prerequisites", lambda **_kwargs: ["Docker is unavailable"])
- result = runner.run_harbor_eval(tmp_path / "demo", ["codex"], progress_reporter=reporter)
+ result = runner.run_harbor_eval(skill, ["codex"], progress_reporter=reporter)
assert result == {"error": ["Docker is unavailable"]}
known_plan = reporter.plans[-1]
@@ -1090,6 +1178,9 @@ def test_runner_does_not_mark_invalid_configuration_ready(
from skillevaluator.tier3.harbor import runner
reporter = _RecordingReporter()
+ skill = tmp_path / "demo"
+ (skill / "evals").mkdir(parents=True)
+ (skill / "evals" / "evals.json").write_text("[]\n", encoding="utf-8")
monkeypatch.setattr(
runner,
"resolve_llm_provider",
@@ -1101,7 +1192,7 @@ def test_runner_does_not_mark_invalid_configuration_ready(
lambda _path: ({"harbor": {"n_attempts": 0}}, None),
)
- result = runner.run_harbor_eval(tmp_path / "demo", ["codex"], progress_reporter=reporter)
+ result = runner.run_harbor_eval(skill, ["codex"], progress_reporter=reporter)
assert result == {"error": ["n_attempts must be >= 1"]}
transitions = [(event.stage, event.state) for event in reporter.events]
@@ -1430,7 +1521,8 @@ def test_command_does_not_relabel_runtime_failure_as_configuration_failure(
from skillevaluator.tier3 import commands
skill = tmp_path / "demo"
- skill.mkdir()
+ (skill / "evals").mkdir(parents=True)
+ (skill / "evals" / "evals.json").write_text("[]\n", encoding="utf-8")
reporter = _RecordingReporter()
monkeypatch.setattr(commands, "resolve_llm_provider", lambda: SimpleNamespace(provider="openai"))
@@ -1602,7 +1694,8 @@ def test_command_runner_terminalizes_inherited_configuration_stage(
from skillevaluator.tier3.harbor import runner
skill = tmp_path / "demo"
- skill.mkdir()
+ (skill / "evals").mkdir(parents=True)
+ (skill / "evals" / "evals.json").write_text("[]\n", encoding="utf-8")
reporter = _RecordingReporter()
monkeypatch.setattr(commands, "resolve_llm_provider", lambda: SimpleNamespace(provider="openai"))
diff --git a/tests/tier3/test_generate_dataset_results.py b/tests/tier3/test_generate_dataset_results.py
index cde672e4..e811da27 100644
--- a/tests/tier3/test_generate_dataset_results.py
+++ b/tests/tier3/test_generate_dataset_results.py
@@ -28,6 +28,7 @@ def test_discover_trajectories_uses_env_results_root(tmp_path, monkeypatch):
trial.mkdir(parents=True)
trajectory = {"steps": [{"type": "assistant", "content": "done"}]}
trial.joinpath("trajectory.json").write_text(json.dumps(trajectory), encoding="utf-8")
+ (run_dir / "run_config.json").write_text("{}", encoding="utf-8")
(run_dir / "result.json").write_text(json.dumps({"run_id": run_id}), encoding="utf-8")
(skill_results / "latest").symlink_to(run_id)
@@ -48,6 +49,7 @@ def test_discover_trajectories_results_dir_overrides_env(tmp_path, monkeypatch):
trial.mkdir(parents=True)
trajectory = {"steps": [{"type": "assistant", "content": "done"}]}
trial.joinpath("trajectory.json").write_text(json.dumps(trajectory), encoding="utf-8")
+ (run_dir / "run_config.json").write_text("{}", encoding="utf-8")
(run_dir / "result.json").write_text(json.dumps({"run_id": run_id}), encoding="utf-8")
(skill_results / "latest").symlink_to(run_id)
diff --git a/tests/utils/test_secure_fs_selected_reads.py b/tests/utils/test_secure_fs_selected_reads.py
new file mode 100644
index 00000000..c5455a8b
--- /dev/null
+++ b/tests/utils/test_secure_fs_selected_reads.py
@@ -0,0 +1,79 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Focused contracts for descriptor-anchored evaluator artifact reads."""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from skillevaluator.utils import secure_fs
+from skillevaluator.utils.secure_fs import SecurePathError, SecureRoot
+
+
+def test_secure_root_binds_declared_root_to_expected_snapshot(tmp_path: Path) -> None:
+ root = tmp_path / "root"
+ root.mkdir()
+ expected = root.lstat()
+ root.rename(tmp_path / "original-root")
+ root.mkdir()
+
+ with pytest.raises(SecurePathError, match=r"changed|snapshot|identity"), SecureRoot(root, expected=expected):
+ pass
+
+
+def test_windows_selected_file_handle_denies_concurrent_write_and_delete(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ selected = tmp_path / "summary.json"
+ selected.write_text("{}", encoding="utf-8")
+ root = SecureRoot(tmp_path)
+ root._windows_root_handles = [100]
+ captured: dict[str, int | str] = {}
+ opened_flags: list[int] = []
+
+ def open_relative(parent_handle: int, name: str, **kwargs: int) -> int:
+ captured.update({"parent_handle": parent_handle, "name": name, **kwargs})
+ return 200
+
+ monkeypatch.setattr(secure_fs, "_windows_open_relative_handle", open_relative)
+ monkeypatch.setattr(secure_fs, "_validate_windows_read_file_handle", lambda *_args: None)
+ monkeypatch.setattr(secure_fs.os, "fstat", lambda _descriptor: selected.lstat())
+ monkeypatch.setattr(secure_fs.os, "O_BINARY", 0x8000, raising=False)
+ monkeypatch.setitem(
+ sys.modules,
+ "msvcrt",
+ SimpleNamespace(open_osfhandle=lambda _handle, flags: opened_flags.append(flags) or 300),
+ )
+
+ assert root._open_windows(Path("summary.json"), None) == 300
+ assert captured["share"] == 0x1 # FILE_SHARE_READ only
+ assert int(opened_flags[0]) & 0x8000 # O_BINARY
+
+
[email protected](os.name != "nt", reason="requires native Windows sharing semantics")
+def test_windows_selected_file_is_immutable_until_reader_handle_closes(tmp_path: Path) -> None:
+ root = tmp_path / "skill"
+ root.mkdir()
+ selected = root / "summary.json"
+ selected.write_text("original", encoding="utf-8")
+
+ with SecureRoot(root) as secure_root:
+ descriptor = secure_root._open_windows(Path("summary.json"), selected.lstat())
+ try:
+ with pytest.raises(OSError):
+ selected.write_text("changed", encoding="utf-8")
+ with pytest.raises(OSError):
+ selected.unlink()
+ finally:
+ os.close(descriptor)
+
+ selected.write_text("changed", encoding="utf-8")
+ selected.unlink()
+ assert not selected.exists()