Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): harden isolation review paths

Christopher Kevina321f788e70e
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 620c83cb..4eb1918a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -157,6 +157,17 @@ jobs:
tests/validators/test_similarity.py
tests/reporting
tests/test_cli.py
+ tests/test_harbor_output_provenance.py
+ tests/test_harbor_runtime_skill_isolation.py
+ tests/test_harbor_secure_copy.py::test_checked_windows_fallback_accepts_crt_descriptor_identity
+ tests/test_harbor_secure_copy.py::test_checked_windows_fallback_verifies_portable_chmod_semantics
+ tests/test_harbor_secure_copy.py::test_native_windows_fallback_copies_tree_and_file
+ tests/test_harbor_secure_copy.py::test_native_windows_tree_fingerprint_binds_writable_mode
+ tests/test_results_location.py
+ tests/test_results_location_legacy_compatibility.py
+ tests/test_tier3_compare.py
+ tests/test_tier3_progress.py
+ tests/utils/test_secure_fs_selected_reads.py
tests/test_oss_packaging.py
native-windows-local-mode:
diff --git a/docs/eval-datasets.mdx b/docs/eval-datasets.mdx
index b710b2fb..004f71eb 100644
--- a/docs/eval-datasets.mdx
+++ b/docs/eval-datasets.mdx
@@ -266,8 +266,10 @@ grading:
Two optional directories shape the world the agent works in.
**`evals/files/`** holds input fixtures — test data, configs, sample
-documents. The entire directory is copied into `/workspace/input/` inside the
-container, so prompts can reference those paths directly.
+documents. An eval entry's `files` string or list selects only those paths
+(relative to `evals/`) for its `/workspace/input/`. An explicit empty list or
+`null` stages no fixture. For backward compatibility, an entry that omits
+`files` receives the entire shared `evals/files/` directory.
**`evals/environment/`** customizes the container itself:
diff --git a/docs/reports.mdx b/docs/reports.mdx
index 571be74e..0b79dc92 100644
--- a/docs/reports.mdx
+++ b/docs/reports.mdx
@@ -104,14 +104,16 @@ With an external root (the first two options), each skill gets its own
subdirectory — runs land under `<dir>/<skill-name>/`, not directly under
`<dir>`.
-Each run creates a directory named by its run ID (`YYYYMMDD_HHMMSS`), and a
-`latest` symlink points at the newest run:
+Each run creates a collision-safe directory named by its run ID
+(`YYYYMMDD_HHMMSS_<pid>_<12-hex-nonce>`). Where symlinks are supported, a
+`latest` symlink points at the newest run; otherwise readers select the newest
+completed run directly:
<FileTree>
- evals/
- results/
- latest/
- - 20260709_141530/
+ - 20260709_141530_12345_a1b2c3d4e5f6/
- result.json
- run_config.json
- attempt_policy.json
@@ -260,7 +262,7 @@ Retained jobs live under `_harbor-jobs/` inside the run directory.
```bash title="Browse retained trajectories"
skillevaluator tier3 evaluate ./my-skill --agents codex --env-mode docker --harbor-keep-jobs
-skillevaluator harbor-view ./my-skill/evals/results/20260709_141530/_harbor-jobs
+skillevaluator harbor-view ./my-skill/evals/results/20260709_141530_12345_a1b2c3d4e5f6/_harbor-jobs
```
(`skillevaluator tier3 harbor-view` is the same command under its expert
diff --git a/docs/tier3-live-evaluation.mdx b/docs/tier3-live-evaluation.mdx
index 724248ed..99a36556 100644
--- a/docs/tier3-live-evaluation.mdx
+++ b/docs/tier3-live-evaluation.mdx
@@ -116,12 +116,13 @@ outside `evals/` by a pre-marker version cannot be identified safely after the
configured location changes. Move or delete that old content before
`--copy-repo` or another full-context evaluation, then rerun with the current
version if replacement evidence is needed. Runtime assets that the agent needs
-should live outside skill-owned `evals/`, under `evals/files/`
-(preferred), or under another non-control subtree such as `evals/data/` and be
-declared as task inputs with `files`. A `files` declaration cannot select
-evaluator control material: dataset/configuration files, graders, developer
-eval guidance, benchmark conversion reports, native Harbor tasks, custom
-environments, evaluator tests, or generated results.
+as part of the installed skill should live outside skill-owned `evals/`. Task
+fixtures should live under `evals/files/` (preferred) or another non-control
+subtree such as `evals/data/`, and should be declared as task inputs with
+`files`. A `files` declaration cannot select evaluator control material:
+dataset/configuration files, graders, developer eval guidance, benchmark
+conversion reports, native Harbor tasks, custom environments, evaluator tests,
+or generated results.
MCP declarations are read only from a bounded, stable, single-link regular
`evals/environment/mcp_servers.toml` through a descriptor-anchored evaluator
@@ -173,7 +174,7 @@ directory:
<FileTree>
- evals/
- results/
- - 20260709_141530/
+ - 20260709_141530_12345_a1b2c3d4e5f6/
- result.json
- run_config.json
- report.html
diff --git a/src/skillevaluator/tier3/harbor/adapter.py b/src/skillevaluator/tier3/harbor/adapter.py
index 8e582524..150657f0 100644
--- a/src/skillevaluator/tier3/harbor/adapter.py
+++ b/src/skillevaluator/tier3/harbor/adapter.py
@@ -35,7 +35,11 @@ from typing import Any
from urllib.parse import unquote
from skillevaluator.tier3.case_ids import safe_child, validate_case_ids, validate_output_directory_path
-from skillevaluator.tier3.harbor.secure_copy import copy_file_secure, copytree_secure
+from skillevaluator.tier3.harbor.secure_copy import (
+ copy_file_secure,
+ copytree_secure,
+ tree_content_fingerprint_secure,
+)
from skillevaluator.tier3.output_provenance import (
GENERATED_OUTPUT_MARKER,
is_generated_output_root,
@@ -112,6 +116,7 @@ _REPO_CONTEXT_IGNORE_PARTS = {("evals", "results")}
_REPO_CONTEXT_IGNORE_SUFFIXES = (".pem", ".key", ".p12", ".pfx")
_NATIVE_SOURCE_IGNORE_NAMES = ("results", "__pycache__", ".git", GENERATED_OUTPUT_MARKER)
_NATIVE_SOURCE_IGNORE = shutil.ignore_patterns(*_NATIVE_SOURCE_IGNORE_NAMES)
+_PATH_DESCRIPTOR_IDENTITIES_COMPARABLE = os.name == "posix"
_REPO_CONTEXT_PUBLIC_ENV_SUFFIXES = (".dist", ".example", ".sample", ".template")
_REPO_CONTEXT_SENSITIVE_NAMES = {
".git-credentials",
@@ -798,6 +803,69 @@ def _check_baseline_skill_candidates_do_not_alias_target(
)
+@dataclass(frozen=True, slots=True)
+class _BaselineAliasValidation:
+ """Run-scoped proof that one exact baseline source set was prevalidated."""
+
+ source_key: tuple[Any, ...]
+
+
+def _baseline_alias_source_key(
+ target_skill: Path,
+ reference_skills_dir: Path | None,
+ workspace_skill_paths: Sequence[Path] | None,
+ excluded_roots: Sequence[Path],
+) -> tuple[Any, ...]:
+ def _path_key(path: Path) -> str:
+ return os.path.normcase(os.fspath(path.expanduser().resolve(strict=False)))
+
+ return (
+ _path_key(target_skill),
+ _path_key(reference_skills_dir) if reference_skills_dir is not None else None,
+ tuple(sorted(_path_key(path) for path in workspace_skill_paths or ())),
+ tuple(sorted(_path_key(path) for path in excluded_roots)),
+ )
+
+
+def _prevalidate_baseline_skill_candidates(
+ target_skill: Path,
+ reference_skills_dir: Path | None,
+ workspace_skill_paths: list[Path] | None,
+ *,
+ excluded_roots: Sequence[Path] = (),
+) -> _BaselineAliasValidation:
+ """Validate configured baseline sources once before a multi-agent run."""
+ _check_baseline_skill_candidates_do_not_alias_target(
+ target_skill,
+ reference_skills_dir,
+ workspace_skill_paths,
+ excluded_roots=excluded_roots,
+ )
+ return _BaselineAliasValidation(
+ _baseline_alias_source_key(
+ target_skill,
+ reference_skills_dir,
+ workspace_skill_paths,
+ excluded_roots,
+ )
+ )
+
+
+def _baseline_alias_validation_matches(
+ validation: _BaselineAliasValidation,
+ target_skill: Path,
+ reference_skills_dir: Path | None,
+ workspace_skill_paths: list[Path] | None,
+ excluded_roots: Sequence[Path],
+) -> bool:
+ return validation.source_key == _baseline_alias_source_key(
+ target_skill,
+ reference_skills_dir,
+ workspace_skill_paths,
+ excluded_roots,
+ )
+
+
def _check_staged_baseline_does_not_contain_target(env_dir: Path, target_skill: Path) -> None:
"""Fail closed if any agent-visible staged file is the evaluated instructions."""
payload = _find_target_manifest_payload(env_dir, target_skill)
@@ -1586,8 +1654,18 @@ def _read_regular_evals_file(
raise ValueError(f"{label} changed while it was read: {path}") from exc
if (
_path_is_link_or_reparse(path, named_after)
- or _evaluator_node_fingerprint(named_after) != _evaluator_node_fingerprint(opened)
or len(payload) != opened.st_size
+ or (
+ _PATH_DESCRIPTOR_IDENTITIES_COMPARABLE
+ and _evaluator_node_fingerprint(named_after) != _evaluator_node_fingerprint(opened)
+ )
+ or (
+ not _PATH_DESCRIPTOR_IDENTITIES_COMPARABLE
+ and (
+ _evaluator_node_fingerprint(named_after) != _evaluator_node_fingerprint(before)
+ or len(payload) != named_after.st_size
+ )
+ )
):
raise ValueError(f"{label} changed while it was read: {path}")
if parent_snapshot:
@@ -3596,6 +3674,7 @@ def _write_dockerfile(
compose_env_names: set[str] | None = None,
repo_context_exclude_paths: Sequence[Path] = (),
agent_workdir: str | None = None,
+ baseline_aliases_prevalidated: bool = False,
) -> None:
"""Generate a Dockerfile that installs skills into the container.
@@ -3608,7 +3687,7 @@ def _write_dockerfile(
env_dir = task_dir / "environment"
env_dir.mkdir(parents=True, exist_ok=True)
- if not has_skill and skill_path is not None:
+ if not has_skill and skill_path is not None and not baseline_aliases_prevalidated:
_check_baseline_skill_candidates_do_not_alias_target(
skill_path,
reference_skills_dir,
@@ -3982,11 +4061,12 @@ def _prepare_native_environment(
compose_env_names: set[str] | None = None,
repo_context_exclude_paths: Sequence[Path] = (),
agent_workdir: str | None = None,
+ baseline_aliases_prevalidated: bool = False,
) -> None:
"""Stage SkillEvaluator runtime additions into a copied native Harbor task."""
env_dir = task_dir / "environment"
env_dir.mkdir(parents=True, exist_ok=True)
- if not has_skill:
+ if not has_skill and not baseline_aliases_prevalidated:
_check_baseline_skill_candidates_do_not_alias_target(
skill_path,
reference_skills_dir,
@@ -4443,6 +4523,7 @@ def _stage_native_harbor_tasks_into(
pre_agent_setup: list[str] | None = None,
task_resources: dict[str, int] | None = None,
agent_workdir: str | None = None,
+ baseline_aliases_prevalidated: bool = False,
) -> list[Path]:
"""Build native Harbor tasks inside a private, caller-owned directory.
@@ -4489,6 +4570,15 @@ def _stage_native_harbor_tasks_into(
)
workspace_skill_names = sorted({p.name for p in workspace_skill_paths or []})
+ effective_excluded_roots = (*repo_context_exclude_paths, *private_repo_context_exclude_paths)
+ if not with_skill and not baseline_aliases_prevalidated:
+ _check_baseline_skill_candidates_do_not_alias_target(
+ skill_path,
+ reference_skills_dir,
+ workspace_skill_paths,
+ excluded_roots=effective_excluded_roots,
+ )
+ baseline_aliases_prevalidated = True
for task_dir in task_dirs:
entry_id = _native_entry_id(task_dir)
native_agent_workdir = _native_task_workdir(task_dir)
@@ -4564,8 +4654,9 @@ def _stage_native_harbor_tasks_into(
grading_mode=grading_mode,
repo_context_mode="full" if copy_repo else "linked",
compose_env_names=set(runtime_env or {}),
- repo_context_exclude_paths=(*repo_context_exclude_paths, *private_repo_context_exclude_paths),
+ repo_context_exclude_paths=effective_excluded_roots,
agent_workdir=native_agent_workdir,
+ baseline_aliases_prevalidated=baseline_aliases_prevalidated,
)
if not (output_dir / "dataset.toml").exists():
@@ -4619,6 +4710,7 @@ def stage_native_harbor_tasks(
task_resources: dict[str, int] | None = None,
agent_workdir: str | None = None,
evaluator_skill_path: Path | None = None,
+ _baseline_alias_validation: _BaselineAliasValidation | None = None,
) -> list[Path]:
"""Stage native tasks privately, then publish one exact output snapshot."""
@@ -4642,8 +4734,21 @@ def stage_native_harbor_tasks(
task_resources=task_resources,
agent_workdir=agent_workdir,
evaluator_skill_path=private_skill_path,
+ _baseline_alias_validation=_baseline_alias_validation,
)
+ baseline_aliases_prevalidated = False
+ if not with_skill and _baseline_alias_validation is not None:
+ if not _baseline_alias_validation_matches(
+ _baseline_alias_validation,
+ skill_path,
+ reference_skills_dir,
+ workspace_skill_paths,
+ repo_context_exclude_paths,
+ ):
+ raise ValueError("Run-scoped baseline alias validation does not match the requested source set")
+ baseline_aliases_prevalidated = True
+
validate_output_provenance_key_location(
skill_path,
output_dir,
@@ -4695,6 +4800,7 @@ def stage_native_harbor_tasks(
pre_agent_setup=pre_agent_setup,
task_resources=task_resources,
agent_workdir=agent_workdir,
+ baseline_aliases_prevalidated=baseline_aliases_prevalidated,
)
relative_tasks = [task.relative_to(private_output) for task in private_tasks]
if output_requires_provenance:
@@ -4752,6 +4858,7 @@ def _generate_harbor_tasks_into(
pre_agent_setup: list[str] | None = None,
task_resources: dict[str, int] | None = None,
agent_workdir: str | None = None,
+ baseline_aliases_prevalidated: bool = False,
) -> list[Path]:
"""Generate Harbor task directories inside a private output directory.
@@ -4815,6 +4922,15 @@ def _generate_harbor_tasks_into(
output_dir.mkdir(parents=True, exist_ok=True)
task_dirs: list[str] = []
task_paths: list[Path] = []
+ effective_excluded_roots = (*repo_context_exclude_paths, *private_repo_context_exclude_paths)
+ if not with_skill and not baseline_aliases_prevalidated:
+ _check_baseline_skill_candidates_do_not_alias_target(
+ skill_path,
+ reference_skills_dir,
+ workspace_skill_paths,
+ excluded_roots=effective_excluded_roots,
+ )
+ baseline_aliases_prevalidated = True
for normalized_entry, case_id in prepared_entries:
task_dir = safe_child(output_dir, case_id)
@@ -4868,8 +4984,9 @@ def _generate_harbor_tasks_into(
repo_context_skill_path=skill_path,
repo_context_mode="full" if copy_repo else "linked",
compose_env_names=set(runtime_env or {}),
- repo_context_exclude_paths=(*repo_context_exclude_paths, *private_repo_context_exclude_paths),
+ repo_context_exclude_paths=effective_excluded_roots,
agent_workdir=agent_workdir,
+ baseline_aliases_prevalidated=baseline_aliases_prevalidated,
)
task_dirs.append(case_id)
@@ -5227,6 +5344,12 @@ def private_evaluator_skill_snapshot(
ignored.update(_NATIVE_SOURCE_IGNORE(current, names))
return ignored
+ source_content_fingerprint = tree_content_fingerprint_secure(
+ source_evals_dir,
+ ignore=_ignore_unselected_evaluator_paths,
+ allowed_root=source_evals_dir,
+ )
+
copytree_secure(
source_evals_dir,
evaluator_skill_path / "evals",
@@ -5240,6 +5363,9 @@ def private_evaluator_skill_snapshot(
)
if copied_selection.signature != selection.signature:
raise ValueError("Evaluator source selection changed while its private snapshot was created")
+ copied_content_fingerprint = tree_content_fingerprint_secure(evaluator_skill_path / "evals")
+ if copied_content_fingerprint != source_content_fingerprint:
+ raise ValueError("Evaluator source selection changed while its private snapshot was created")
yield evaluator_skill_path
@@ -5262,6 +5388,7 @@ def generate_harbor_tasks(
task_resources: dict[str, int] | None = None,
agent_workdir: str | None = None,
evaluator_skill_path: Path | None = None,
+ _baseline_alias_validation: _BaselineAliasValidation | None = None,
) -> list[Path]:
"""Generate tasks from one private evals snapshot, then publish exactly."""
@@ -5287,10 +5414,23 @@ def generate_harbor_tasks(
task_resources=task_resources,
agent_workdir=agent_workdir,
evaluator_skill_path=private_skill_path,
+ _baseline_alias_validation=_baseline_alias_validation,
)
if find_evals_file(evaluator_skill_path) is None:
raise FileNotFoundError(f"No evals dataset found in {evaluator_skill_path / 'evals'}")
+ baseline_aliases_prevalidated = False
+ if not with_skill and _baseline_alias_validation is not None:
+ if not _baseline_alias_validation_matches(
+ _baseline_alias_validation,
+ skill_path,
+ reference_skills_dir,
+ workspace_skill_paths,
+ repo_context_exclude_paths,
+ ):
+ raise ValueError("Run-scoped baseline alias validation does not match the requested source set")
+ baseline_aliases_prevalidated = True
+
validate_output_provenance_key_location(
skill_path,
output_dir,
@@ -5342,6 +5482,7 @@ def generate_harbor_tasks(
pre_agent_setup=pre_agent_setup,
task_resources=task_resources,
agent_workdir=agent_workdir,
+ baseline_aliases_prevalidated=baseline_aliases_prevalidated,
)
relative_tasks = [task.relative_to(private_output) for task in private_tasks]
if output_requires_provenance:
diff --git a/src/skillevaluator/tier3/harbor/runner.py b/src/skillevaluator/tier3/harbor/runner.py
index b0ff319e..2c08f3d7 100644
--- a/src/skillevaluator/tier3/harbor/runner.py
+++ b/src/skillevaluator/tier3/harbor/runner.py
@@ -18,7 +18,7 @@ import time
import tomllib
from collections.abc import Iterator, Mapping
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, as_completed, wait
-from contextlib import ExitStack, contextmanager, suppress
+from contextlib import ExitStack, contextmanager
from dataclasses import dataclass
from datetime import UTC, datetime
from functools import wraps
@@ -32,6 +32,7 @@ from skillevaluator.evaluation.tier3_report import render_agent_eval_html_report
from skillevaluator.provider_config import ProviderConfig, ProviderConfigurationError, resolve_llm_provider
from skillevaluator.tier3.evals_config import EvalsConfigError, load_evals_config
from skillevaluator.tier3.harbor.adapter import (
+ _prevalidate_baseline_skill_candidates,
build_eval_base_image,
find_evals_file,
generate_harbor_tasks,
@@ -58,7 +59,12 @@ from skillevaluator.tier3.harbor.progress import (
)
from skillevaluator.tier3.harbor.secure_copy import copytree_secure
from skillevaluator.tier3.harbor.secure_docker_environment import SECURE_DOCKER_ENV_IMPORT_PATH
-from skillevaluator.tier3.output_provenance import mark_generated_output_root
+from skillevaluator.tier3.output_provenance import (
+ mark_generated_output_root,
+ remove_generated_output_root_if_owned,
+ remove_output_reservation_if_identity_matches,
+ write_output_file_atomically,
+)
from skillevaluator.tier3.results_location import publish_latest_results
from skillevaluator.tier3_environments import DEFAULT_ENV_MODE, ENV_MODE_LOCAL, HARBOR_ENV_MODES
@@ -79,11 +85,13 @@ def _reserve_run_dir(results_root: Path, timestamp: str) -> Path:
run_dir.mkdir()
except FileExistsError:
continue
+ reservation_metadata = run_dir.lstat()
+ reservation_identity = reservation_metadata.st_dev, reservation_metadata.st_ino
try:
mark_generated_output_root(run_dir)
except Exception:
- with suppress(OSError):
- run_dir.rmdir()
+ if not remove_generated_output_root_if_owned(run_dir, expected_identity=reservation_identity):
+ remove_output_reservation_if_identity_matches(run_dir, reservation_identity)
raise
return run_dir
raise RuntimeError("Could not reserve a unique Tier 3 run directory")
@@ -1874,20 +1882,35 @@ def _run_harbor_eval_impl(
tasks_dir = run_dir / "_harbor-tasks"
result_path = run_dir / "result.json"
report_path: Path | None = None
- jobs_dir.mkdir(parents=True, exist_ok=True)
- def _emit_run_finished(state: str, detail: str) -> None:
+ def _emit_run_finished(state: str, detail: str, *, include_artifacts: bool = True) -> None:
reporter.emit(
ProgressEvent(
stage="run-finished",
state=state,
detail=detail,
- output_dir=str(run_dir),
- result_path=str(result_path) if result_path.is_file() else None,
- report_path=str(report_path) if report_path is not None and report_path.is_file() else None,
+ output_dir=str(run_dir) if include_artifacts else None,
+ result_path=str(result_path) if include_artifacts and result_path.is_file() else None,
+ report_path=(
+ str(report_path)
+ if include_artifacts and report_path is not None and report_path.is_file()
+ else None
+ ),
)
)
+ reservation_identity: tuple[int, int] | None = None
+ try:
+ reservation_metadata = run_dir.lstat()
+ reservation_identity = reservation_metadata.st_dev, reservation_metadata.st_ino
+ jobs_dir.mkdir(parents=True, exist_ok=True)
+ except OSError as exc:
+ reporter.emit(ProgressEvent(stage="with-skill-tasks", state="failed", detail=str(exc)))
+ if reservation_identity is not None:
+ remove_generated_output_root_if_owned(run_dir, expected_identity=reservation_identity)
+ _emit_run_finished("failed", "Harbor jobs directory could not be created", include_artifacts=False)
+ return {"error": [str(exc)]}
+
emitter = stage_native_harbor_tasks if task_source == "native_harbor" else generate_harbor_tasks
resource_config = harbor_config.get("resources", {})
use_base_image = env_mode == "docker" and base_image_mode != "disabled"
@@ -1964,6 +1987,14 @@ def _run_harbor_eval_impl(
if not skip_baseline:
reporter.emit(ProgressEvent(stage="baseline-tasks", state="running"))
staging_failure_stage = "baseline-tasks"
+ baseline_alias_validation = _prevalidate_baseline_skill_candidates(
+ skill_path,
+ reference_skills_dir,
+ workspace_skills,
+ excluded_roots=(root,),
+ )
+ else:
+ baseline_alias_validation = None
for agent in agents:
without_dir = agent_task_dirs[agent][1]
if without_dir is not None:
@@ -1985,6 +2016,7 @@ def _run_harbor_eval_impl(
task_resources=resource_config,
agent_workdir=harbor_config.get("agent_workdir"),
evaluator_skill_path=evaluator_skill_path,
+ _baseline_alias_validation=baseline_alias_validation,
)
if not skip_baseline:
reporter.emit(ProgressEvent(stage="baseline-tasks", state="ready", detail="baseline inputs staged"))
@@ -2073,7 +2105,7 @@ def _run_harbor_eval_impl(
"result_path": str(result_path),
"agents": {},
}
- result_path.write_text(json.dumps(failed_result, indent=2), encoding="utf-8")
+ write_output_file_atomically(result_path, json.dumps(failed_result, indent=2).encode("utf-8"))
_emit_run_finished("failed", "agent runtime preflight failed")
return failed_result
reporter.emit(
@@ -2266,7 +2298,7 @@ def _run_harbor_eval_impl(
results["duration_seconds"] = round(time.monotonic() - started_at, 3)
try:
- result_path.write_text(json.dumps(results, indent=2), encoding="utf-8")
+ write_output_file_atomically(result_path, json.dumps(results, indent=2).encode("utf-8"))
except Exception:
reporter.emit(ProgressEvent(stage="report", state="failed", detail="result write failed"))
_emit_run_finished("failed", "report artifacts could not be written")
@@ -2326,7 +2358,7 @@ def _finalize_harbor_artifacts(
result_path_value = result.get("result_path")
result_path = Path(str(result_path_value)) if result_path_value else run_dir / "result.json"
if result_path.is_file():
- result_path.write_text(json.dumps(result, indent=2), encoding="utf-8")
+ write_output_file_atomically(result_path, json.dumps(result, indent=2).encode("utf-8"))
run_config = result.get("run_config")
run_config_path = run_dir / "run_config.json"
if isinstance(run_config, dict) and run_config_path.is_file():
diff --git a/src/skillevaluator/tier3/harbor/secure_copy.py b/src/skillevaluator/tier3/harbor/secure_copy.py
index fe89d4e8..de450005 100644
--- a/src/skillevaluator/tier3/harbor/secure_copy.py
+++ b/src/skillevaluator/tier3/harbor/secure_copy.py
@@ -39,6 +39,7 @@ _REPARSE_POINT = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
_DIRECTORY_FLAGS = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
_FILE_FLAGS = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_NOCTTY", 0)
_WINDOWS_CHMOD_SEMANTICS = os.name == "nt"
+_PATH_DESCRIPTOR_IDENTITIES_COMPARABLE = os.name == "posix"
_DESCRIPTOR_BACKEND = (
os.name == "posix"
and hasattr(os, "O_NOFOLLOW")
@@ -211,6 +212,25 @@ def _node_identity(metadata: os.stat_result) -> tuple[int, int, int, int, int]:
)
+def _fallback_opened_matches_named(opened: os.stat_result, named: os.stat_result) -> bool:
+ """Compare fallback file metadata without assuming Windows stat identities."""
+ if _PATH_DESCRIPTOR_IDENTITIES_COMPARABLE:
+ return _fingerprint(opened) == _fingerprint(named)
+ return (
+ stat.S_IFMT(opened.st_mode) == stat.S_IFMT(named.st_mode)
+ and opened.st_nlink == named.st_nlink
+ and opened.st_size == named.st_size
+ )
+
+
+def _portable_fingerprint_mode(mode: int) -> int:
+ """Normalize mode bits to the semantics the active copier preserves."""
+ normalized = stat.S_IMODE(mode) & 0o777
+ if _WINDOWS_CHMOD_SEMANTICS:
+ return int(bool(normalized & stat.S_IWRITE))
+ return normalized
+
+
def _entry_from_stat(
parts: tuple[str, ...],
kind: str,
@@ -501,8 +521,13 @@ def _open_fallback_regular(
raise UnsafeStagingError(f"cannot safely open {role} file {path}: {exc}") from exc
try:
opened = os.fstat(descriptor)
- _validate_type(opened, path=path, role=role, root_device=root_device)
- if _fingerprint(opened) != _fingerprint(before):
+ _validate_type(
+ opened,
+ path=path,
+ role=role,
+ root_device=root_device if _PATH_DESCRIPTOR_IDENTITIES_COMPARABLE else None,
+ )
+ if not _fallback_opened_matches_named(opened, before):
raise _changed(path, "file was replaced")
return descriptor
except BaseException:
@@ -513,6 +538,7 @@ def _open_fallback_regular(
def _hash_path_checked(path: Path, before: os.stat_result, *, role: str, root_device: int) -> str:
descriptor = _open_fallback_regular(path, before, role=role, root_device=root_device)
try:
+ opened = os.fstat(descriptor)
digest = _hash_descriptor(descriptor)
after = os.fstat(descriptor)
finally:
@@ -521,7 +547,11 @@ def _hash_path_checked(path: Path, before: os.stat_result, *, role: str, root_de
named = path.lstat()
except OSError as exc:
raise _changed(path, "file disappeared after reading") from exc
- if _fingerprint(after) != _fingerprint(before) or _fingerprint(named) != _fingerprint(before):
+ if (
+ _fingerprint(after) != _fingerprint(opened)
+ or _fingerprint(named) != _fingerprint(before)
+ or not _fallback_opened_matches_named(after, named)
+ ):
raise _changed(path, "file changed while it was validated")
return digest
@@ -608,6 +638,32 @@ def _build_tree_manifest(
return _build_tree_manifest_fallback(source_path, root_path, role=role, ignore=ignore)
+def tree_content_fingerprint_secure(
+ source: Path | str,
+ *,
+ allowed_root: Path | str | None = None,
+ ignore: IgnoreCallback | None = None,
+) -> str:
+ """Return a deterministic content fingerprint for one securely validated tree.
+
+ Filesystem identity and timestamps intentionally do not participate: a
+ securely copied tree has different inodes and may have different directory
+ metadata. Relative paths, node kinds, and regular-file bytes do, so the
+ result can bind source selection to the exact tree later staged elsewhere.
+ """
+ manifest = _build_tree_manifest(source, allowed_root=allowed_root, ignore=ignore)
+ digest = hashlib.sha256()
+ for entry in sorted((manifest.root, *manifest.entries), key=lambda item: item.parts):
+ path = os.fsencode("/".join(entry.parts))
+ kind = entry.kind.encode("ascii")
+ mode = _portable_fingerprint_mode(entry.mode).to_bytes(2, "big")
+ content = (entry.digest or "").encode("ascii")
+ for field in (path, kind, mode, content):
+ digest.update(len(field).to_bytes(8, "big"))
+ digest.update(field)
+ return digest.hexdigest()
+
+
def _verify_entry(entry: _ManifestEntry, metadata: os.stat_result, path: Path) -> None:
if _is_link(metadata):
raise UnsafeStagingError(f"source changed to a symlink or reparse point: {path}")
@@ -1568,7 +1624,7 @@ def _apply_fallback_open_file_mode(path: Path, descriptor: int, mode: int) -> No
or not stat.S_ISREG(named_before.st_mode)
or opened_before.st_nlink != 1
or named_before.st_nlink != 1
- or _node_identity(named_before) != _node_identity(opened_before)
+ or not _fallback_opened_matches_named(opened_before, named_before)
):
raise UnsafeStagingError("fallback staging file changed before its mode was applied")
fchmod = getattr(os, "fchmod", None)
@@ -1581,7 +1637,8 @@ def _apply_fallback_open_file_mode(path: Path, descriptor: int, mode: int) -> No
if (
_is_link(named_after)
or _node_identity(opened_after) != _node_identity(opened_before)
- or _node_identity(named_after) != _node_identity(opened_after)
+ or _node_identity(named_after) != _node_identity(named_before)
+ or not _fallback_opened_matches_named(opened_after, named_after)
or not _fallback_mode_matches(opened_after.st_mode, mode)
or not _fallback_mode_matches(named_after.st_mode, mode)
):
@@ -1606,6 +1663,7 @@ def _copy_manifest_file_fallback(manifest: _TreeManifest, entry: _ManifestEntry,
)
destination_descriptor = -1
try:
+ source_opened = os.fstat(source_descriptor)
destination_descriptor = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
_apply_fallback_open_file_mode(destination, destination_descriptor, 0o600)
digest = hashlib.sha256()
@@ -1615,8 +1673,13 @@ def _copy_manifest_file_fallback(manifest: _TreeManifest, entry: _ManifestEntry,
while view:
written = os.write(destination_descriptor, view)
view = view[written:]
- _verify_entry(entry, os.fstat(source_descriptor), source)
- _verify_entry(entry, source.lstat(), source)
+ source_after = os.fstat(source_descriptor)
+ named_after = source.lstat()
+ if _fingerprint(source_after) != _fingerprint(source_opened) or not _fallback_opened_matches_named(
+ source_after, named_after
+ ):
+ raise _changed(source, "file changed while it was copied")
+ _verify_entry(entry, named_after, source)
if digest.hexdigest() != entry.digest:
raise _changed(source, "file contents changed")
finally:
@@ -2137,6 +2200,7 @@ def _stage_fallback_file(manifest: _FileManifest, stage: Path) -> None:
)
destination_descriptor = -1
try:
+ source_opened = os.fstat(source_descriptor)
destination_descriptor = os.open(stage, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
_apply_fallback_open_file_mode(stage, destination_descriptor, 0o600)
digest = hashlib.sha256()
@@ -2146,8 +2210,13 @@ def _stage_fallback_file(manifest: _FileManifest, stage: Path) -> None:
while view:
written = os.write(destination_descriptor, view)
view = view[written:]
- _verify_entry(manifest.entry, os.fstat(source_descriptor), manifest.source)
- _verify_entry(manifest.entry, manifest.source.lstat(), manifest.source)
+ source_after = os.fstat(source_descriptor)
+ named_after = manifest.source.lstat()
+ if _fingerprint(source_after) != _fingerprint(source_opened) or not _fallback_opened_matches_named(
+ source_after, named_after
+ ):
+ raise _changed(manifest.source, "file changed while it was copied")
+ _verify_entry(manifest.entry, named_after, manifest.source)
if digest.hexdigest() != manifest.entry.digest:
raise _changed(manifest.source, "file contents changed")
finally:
diff --git a/src/skillevaluator/tier3/output_provenance.py b/src/skillevaluator/tier3/output_provenance.py
index 4949abb1..86adaac1 100644
--- a/src/skillevaluator/tier3/output_provenance.py
+++ b/src/skillevaluator/tier3/output_provenance.py
@@ -16,6 +16,8 @@ import time
from pathlib import Path
from skillevaluator.tier3.case_ids import validate_output_directory_path
+from skillevaluator.tier3.harbor.secure_copy import _DESCRIPTOR_BACKEND, _DIRECTORY_FLAGS, _remove_tree_at
+from skillevaluator.utils.secure_fs import SecurePathError, SecureRoot
GENERATED_OUTPUT_MARKER = ".skillevaluator-generated-output"
OUTPUT_PROVENANCE_KEY_ENV = "SKILLEVALUATOR_OUTPUT_PROVENANCE_KEY_FILE"
@@ -26,6 +28,7 @@ _KEY_TEMP_PREFIX = ".output-provenance.key.tmp-"
_MARKER_PREFIX = b"SkillEvaluator generated output v2\n"
_MARKER_CONTEXT = b"skillevaluator.generated-output.v2\0"
_MARKER_SIZE = len(_MARKER_PREFIX) + len(base64.urlsafe_b64encode(bytes(_KEY_BYTES)).rstrip(b"=")) + 1
+_PATH_DESCRIPTOR_IDENTITIES_COMPARABLE = os.name == "posix"
def _is_link_or_reparse(metadata: os.stat_result) -> bool:
@@ -37,6 +40,157 @@ def _node_fingerprint(metadata: os.stat_result) -> tuple[int, int, int, int, int
return metadata.st_dev, metadata.st_ino, metadata.st_mode, metadata.st_nlink, metadata.st_size
+def _artifact_fingerprint(metadata: os.stat_result) -> tuple[int, int, int, int, int, int, int]:
+ """Return identity and content-change fields from one stat API family."""
+ return (
+ metadata.st_dev,
+ metadata.st_ino,
+ metadata.st_mode,
+ metadata.st_nlink,
+ metadata.st_size,
+ metadata.st_mtime_ns,
+ metadata.st_ctime_ns,
+ )
+
+
+def _stable_read_metadata(
+ before: os.stat_result,
+ opened: os.stat_result,
+ after: os.stat_result,
+) -> os.stat_result | None:
+ """Select comparable metadata after a descriptor-pinned read."""
+ if _PATH_DESCRIPTOR_IDENTITIES_COMPARABLE:
+ if _artifact_fingerprint(opened) != _artifact_fingerprint(before) or _artifact_fingerprint(
+ after
+ ) != _artifact_fingerprint(opened):
+ return None
+ return opened
+ if _artifact_fingerprint(after) != _artifact_fingerprint(before) or opened.st_size != after.st_size:
+ return None
+ return after
+
+
+def _fsync_directory(path: Path) -> None:
+ """Durably publish a directory entry where the platform supports it."""
+ if os.name != "posix":
+ return
+ descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
+ try:
+ os.fsync(descriptor)
+ finally:
+ os.close(descriptor)
+
+
+def _unlink_if_same_file(path: Path, expected: os.stat_result) -> bool:
+ """Unlink only the exact regular file created by the current operation."""
+ try:
+ observed = path.lstat()
+ except OSError:
+ return False
+ if _is_link_or_reparse(observed) or not stat.S_ISREG(observed.st_mode):
+ return False
+ if not os.path.samestat(expected, observed):
+ return False
+ try:
+ path.unlink()
+ except OSError:
+ return False
+ return True
+
+
+def _inspect_atomic_destination(path: Path) -> os.stat_result | None:
+ """Return destination metadata after rejecting unsafe file types."""
+ try:
+ metadata = path.lstat()
+ except FileNotFoundError:
+ return None
+ if _is_link_or_reparse(metadata) or not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1:
+ raise ValueError(f"Generated output artifact must be a single-link regular file: {path}")
+ return metadata
+
+
+def write_output_file_atomically(path: Path, payload: bytes) -> None:
+ """Durably replace one evaluator-owned output file with complete bytes."""
+ validate_output_directory_path(path.parent)
+ existing = _inspect_atomic_destination(path)
+ existing_mode = stat.S_IMODE(existing.st_mode) if existing is not None else None
+ temporary: Path | None = None
+ temporary_named_metadata: os.stat_result | None = None
+ descriptor = -1
+ try:
+ for _ in range(32):
+ candidate = path.parent / f".{path.name}.{os.getpid()}-{secrets.token_hex(8)}.tmp"
+ try:
+ descriptor = os.open(
+ candidate,
+ os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0),
+ 0o666,
+ )
+ except FileExistsError:
+ continue
+ temporary = candidate
+ break
+ if temporary is None or descriptor < 0:
+ raise OSError(f"Cannot allocate a temporary output artifact for: {path}")
+
+ temporary_opened = os.fstat(descriptor)
+ temporary_named_metadata = temporary.lstat()
+ if (
+ _is_link_or_reparse(temporary_opened)
+ or _is_link_or_reparse(temporary_named_metadata)
+ or not stat.S_ISREG(temporary_opened.st_mode)
+ or not stat.S_ISREG(temporary_named_metadata.st_mode)
+ or temporary_opened.st_nlink != 1
+ or temporary_named_metadata.st_nlink != 1
+ or (
+ _PATH_DESCRIPTOR_IDENTITIES_COMPARABLE
+ and not os.path.samestat(temporary_opened, temporary_named_metadata)
+ )
+ ):
+ raise ValueError(f"Generated output temporary artifact is unsafe: {temporary}")
+ if existing_mode is not None and hasattr(os, "fchmod"):
+ os.fchmod(descriptor, existing_mode)
+
+ handle = os.fdopen(descriptor, "wb", closefd=True)
+ descriptor = -1
+ with handle:
+ if handle.write(payload) != len(payload):
+ raise OSError(f"Short write while publishing generated output: {path}")
+ handle.flush()
+ os.fsync(handle.fileno())
+ written = os.fstat(handle.fileno())
+
+ observed = temporary.lstat()
+ temporary_identity_matches = os.path.samestat(temporary_named_metadata, observed) and (
+ not _PATH_DESCRIPTOR_IDENTITIES_COMPARABLE or os.path.samestat(written, observed)
+ )
+ if (
+ _is_link_or_reparse(observed)
+ or not stat.S_ISREG(observed.st_mode)
+ or observed.st_nlink != 1
+ or written.st_size != len(payload)
+ or observed.st_size != len(payload)
+ or not temporary_identity_matches
+ ):
+ raise ValueError(f"Generated output temporary artifact changed while writing: {temporary}")
+ validate_output_directory_path(path.parent)
+ destination = _inspect_atomic_destination(path)
+ if (existing is None) != (destination is None) or (
+ existing is not None
+ and destination is not None
+ and _artifact_fingerprint(existing) != _artifact_fingerprint(destination)
+ ):
+ raise ValueError(f"Generated output destination changed while writing: {path}")
+ os.replace(temporary, path) # noqa: PTH105 -- same-directory atomic replacement is the contract
+ temporary = None
+ _fsync_directory(path.parent)
+ finally:
+ if descriptor >= 0:
+ os.close(descriptor)
+ if temporary is not None and temporary_named_metadata is not None:
+ _unlink_if_same_file(temporary, temporary_named_metadata)
+
+
def _protect_key_for_storage(key: bytes) -> bytes:
"""Bind key bytes to the current Windows user with DPAPI."""
if os.name != "nt":
@@ -149,21 +303,16 @@ def _validate_key_metadata(path: Path, metadata: os.stat_result) -> None:
def _read_key(path: Path) -> bytes:
before = path.lstat()
_validate_key_metadata(path, before)
- flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)
- descriptor = os.open(path, flags)
try:
- opened = os.fstat(descriptor)
- _validate_key_metadata(path, opened)
- if _node_fingerprint(opened) != _node_fingerprint(before):
- raise ValueError(f"Output provenance key changed while it was opened: {path}")
- payload = os.read(descriptor, before.st_size + 1)
- after = os.fstat(descriptor)
- _validate_key_metadata(path, after)
- if _node_fingerprint(after) != _node_fingerprint(before):
- raise ValueError(f"Output provenance key changed while it was read: {path}")
- finally:
- os.close(descriptor)
- if len(payload) != before.st_size:
+ with SecureRoot(path.parent) as secure_root:
+ payload, opened = secure_root.read_bytes(Path(path.name), before.st_size, expected=before)
+ after = path.lstat()
+ except SecurePathError as exc:
+ raise ValueError(f"Output provenance key changed while it was read: {path}") from exc
+ _validate_key_metadata(path, opened)
+ _validate_key_metadata(path, after)
+ stable = _stable_read_metadata(before, opened, after)
+ if stable is None or len(payload) != before.st_size or len(payload) != stable.st_size:
raise ValueError(f"Output provenance key has an invalid size: {path}")
return _unprotect_stored_key(payload)
@@ -344,30 +493,26 @@ def _read_marker(path: Path, expected_size: int) -> bytes | None:
or before.st_size != expected_size
):
return None
- flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)
- try:
- descriptor = os.open(path, flags)
- except OSError:
- return None
try:
- opened = os.fstat(descriptor)
+ with SecureRoot(path.parent) as secure_root:
+ payload, opened = secure_root.read_bytes(Path(path.name), expected_size, expected=before)
+ after = path.lstat()
if (
_is_link_or_reparse(opened)
or not stat.S_ISREG(opened.st_mode)
or opened.st_nlink != 1
or opened.st_size != expected_size
- or _node_fingerprint(opened) != _node_fingerprint(before)
+ or _is_link_or_reparse(after)
+ or not stat.S_ISREG(after.st_mode)
+ or after.st_nlink != 1
+ or after.st_size != expected_size
+ or _stable_read_metadata(before, opened, after) is None
+ or len(payload) != expected_size
):
return None
- payload = os.read(descriptor, expected_size + 1)
- after = os.fstat(descriptor)
- if _node_fingerprint(after) != _node_fingerprint(before):
- return None
return payload
- except OSError:
+ except (OSError, SecurePathError, ValueError):
return None
- finally:
- os.close(descriptor)
def is_generated_output_root(path: Path) -> bool:
@@ -420,10 +565,50 @@ def write_generated_output_marker(root: Path, *, destination: Path | None = None
if observed is None or not hmac.compare_digest(observed, expected):
raise ValueError(f"Generated output marker is invalid or unsafe: {marker}") from None
return
- with os.fdopen(descriptor, "wb", closefd=True) as handle:
- handle.write(expected)
- handle.flush()
- os.fsync(handle.fileno())
+ created_named: os.stat_result | None = None
+ try:
+ created_opened = os.fstat(descriptor)
+ created_named = marker.lstat()
+ if (
+ _is_link_or_reparse(created_opened)
+ or _is_link_or_reparse(created_named)
+ or not stat.S_ISREG(created_opened.st_mode)
+ or not stat.S_ISREG(created_named.st_mode)
+ or created_opened.st_nlink != 1
+ or created_named.st_nlink != 1
+ or (_PATH_DESCRIPTOR_IDENTITIES_COMPARABLE and not os.path.samestat(created_opened, created_named))
+ ):
+ raise ValueError(f"Generated output marker is unsafe: {marker}")
+ handle = os.fdopen(descriptor, "wb", closefd=True)
+ descriptor = -1
+ with handle:
+ if handle.write(expected) != len(expected):
+ raise OSError(f"Short write while publishing generated output marker: {marker}")
+ handle.flush()
+ os.fsync(handle.fileno())
+ written = os.fstat(handle.fileno())
+ observed = marker.lstat()
+ if (
+ _is_link_or_reparse(observed)
+ or not stat.S_ISREG(observed.st_mode)
+ or observed.st_nlink != 1
+ or written.st_size != len(expected)
+ or observed.st_size != len(expected)
+ or not os.path.samestat(created_named, observed)
+ or (_PATH_DESCRIPTOR_IDENTITIES_COMPARABLE and not os.path.samestat(written, observed))
+ ):
+ raise ValueError(f"Generated output marker changed while it was written: {marker}")
+ _fsync_directory(root)
+ except BaseException:
+ if descriptor >= 0:
+ os.close(descriptor)
+ descriptor = -1
+ if created_named is not None:
+ _unlink_if_same_file(marker, created_named)
+ raise
+ finally:
+ if descriptor >= 0:
+ os.close(descriptor)
def mark_generated_output_root(path: Path) -> None:
@@ -432,3 +617,115 @@ def mark_generated_output_root(path: Path) -> None:
path.mkdir(parents=True, exist_ok=True)
validate_output_directory_path(path)
write_generated_output_marker(path)
+
+
+def _remove_output_tree_at_identity(path: Path, expected_identity: tuple[int, int]) -> bool:
+ """Remove only one no-follow tree matching a previously observed identity."""
+ if not _DESCRIPTOR_BACKEND:
+ return False
+ parent_descriptor = -1
+ try:
+ validate_output_directory_path(path.parent)
+ parent_descriptor = os.open(path.parent, _DIRECTORY_FLAGS)
+ _remove_tree_at(parent_descriptor, path.name, expected_identity=expected_identity)
+ os.fsync(parent_descriptor)
+ except (OSError, ValueError):
+ return False
+ finally:
+ if parent_descriptor >= 0:
+ os.close(parent_descriptor)
+ return not os.path.lexists(path)
+
+
+def _remove_empty_output_tree_fallback(
+ path: Path,
+ expected_identity: tuple[int, int],
+ *,
+ remove_authenticated_marker: bool,
+) -> bool:
+ """Conservatively remove an empty exact reservation without recursive deletion."""
+ marker = path / GENERATED_OUTPUT_MARKER
+ marker_metadata: os.stat_result | None = None
+ try:
+ validate_output_directory_path(path)
+ before = path.lstat()
+ if (
+ _is_link_or_reparse(before)
+ or not stat.S_ISDIR(before.st_mode)
+ or (before.st_dev, before.st_ino) != expected_identity
+ ):
+ return False
+ entries = list(path.iterdir())
+ if remove_authenticated_marker:
+ if len(entries) != 1 or entries[0].name != GENERATED_OUTPUT_MARKER or not is_generated_output_root(path):
+ return False
+ marker_metadata = marker.lstat()
+ if (
+ _is_link_or_reparse(marker_metadata)
+ or not stat.S_ISREG(marker_metadata.st_mode)
+ or marker_metadata.st_nlink != 1
+ ):
+ return False
+ elif entries:
+ return False
+
+ current = path.lstat()
+ if (current.st_dev, current.st_ino) != expected_identity or _is_link_or_reparse(current):
+ return False
+ if marker_metadata is not None and not _unlink_if_same_file(marker, marker_metadata):
+ return False
+ current = path.lstat()
+ if (current.st_dev, current.st_ino) != expected_identity or _is_link_or_reparse(current):
+ return False
+ if next(path.iterdir(), None) is not None:
+ return False
+ path.rmdir()
+ _fsync_directory(path.parent)
+ except (OSError, ValueError):
+ return False
+ return not os.path.lexists(path)
+
+
+def remove_output_reservation_if_identity_matches(path: Path, expected_identity: tuple[int, int]) -> bool:
+ """Clean an exact new reservation; fallback platforms require it to be empty."""
+ if not _DESCRIPTOR_BACKEND:
+ return _remove_empty_output_tree_fallback(
+ path,
+ expected_identity,
+ remove_authenticated_marker=False,
+ )
+ return _remove_output_tree_at_identity(path, expected_identity)
+
+
+def remove_generated_output_root_if_owned(
+ path: Path,
+ *,
+ expected_identity: tuple[int, int] | None = None,
+) -> bool:
+ """Remove an authenticated failed reservation without following replacements.
+
+ Descriptor-capable platforms use identity-bound recursive removal. Other
+ platforms remove only a marker-only directory and never recursively delete.
+ """
+ try:
+ validate_output_directory_path(path)
+ before = path.lstat()
+ if not stat.S_ISDIR(before.st_mode) or _is_link_or_reparse(before):
+ return False
+ identity = before.st_dev, before.st_ino
+ if expected_identity is not None and identity != expected_identity:
+ return False
+ if not is_generated_output_root(path):
+ return False
+ after = path.lstat()
+ if _node_fingerprint(after) != _node_fingerprint(before):
+ return False
+ except (OSError, ValueError):
+ return False
+ if not _DESCRIPTOR_BACKEND:
+ return _remove_empty_output_tree_fallback(
+ path,
+ identity,
+ remove_authenticated_marker=True,
+ )
+ return _remove_output_tree_at_identity(path, identity)
diff --git a/src/skillevaluator/tier3/results_location.py b/src/skillevaluator/tier3/results_location.py
index ac5ef4d1..09c30fb5 100644
--- a/src/skillevaluator/tier3/results_location.py
+++ b/src/skillevaluator/tier3/results_location.py
@@ -8,6 +8,7 @@ from __future__ import annotations
import json
import math
import os
+import re
import stat
import subprocess
from contextlib import suppress
@@ -15,12 +16,16 @@ from datetime import datetime
from pathlib import Path
from uuid import uuid4
-from skillevaluator.tier3.output_provenance import GENERATED_OUTPUT_MARKER
+from skillevaluator.tier3.output_provenance import GENERATED_OUTPUT_MARKER, is_generated_output_root
from skillevaluator.utils.secure_fs import SecurePathError, SecureRoot
ENV_RESULTS_DIR = "SKILLEVALUATOR_RESULTS_DIR"
_RUN_TIMESTAMP_FORMATS = (("%Y%m%d_%H%M%S", 15), ("%Y-%m-%d_%H%M%S", 17))
+# Pre-provenance releases emitted exact timestamp names. The collision-safe
+# PID/UUID suffix and run-level marker were introduced together, so a suffix is
+# a reliable format boundary and must never fall back to the historical parser.
+_CURRENT_RUN_SUFFIX = re.compile(r"_[1-9][0-9]{0,19}_[0-9a-f]{12}\Z")
# A current completed run carries ``run_config.json`` plus an atomically
# written ``result.json`` whose run identity matches its directory. Historical
@@ -32,6 +37,10 @@ _FINAL_RESULT_ARTIFACT = "result.json"
# The writer emits a small enumeration; this generous cap rejects pathological
# metadata without relying on Python's bounded decimal-to-int conversion.
_MAX_LEGACY_OCCURRENCE_DIGITS = 32
+_CURRENT_BASE_IMAGE_MODES = frozenset({"reuse", "rebuild", "disabled"})
+_CURRENT_GRADING_MODES = frozenset({"default", "default_plus_custom", "custom_only"})
+_CURRENT_EXECUTION_STATUSES = frozenset({"succeeded", "failed", "skipped"})
+_PATH_DESCRIPTOR_IDENTITIES_COMPARABLE = os.name == "posix"
# These runtime-owned directories predate canonical run-level ``result.json``;
# ``staged/`` arrived with that artifact and is intentionally not legacy-safe.
_LEGACY_RUNTIME_SIDECAR_DIRS = frozenset({"harbor-run-logs", "astra-cleanup"})
@@ -112,20 +121,27 @@ def iter_candidate_results_roots(
return roots
-def _run_timestamp(name: str) -> datetime | None:
- """Parse the timestamp prefix shared by current and legacy run names."""
+def _run_timestamp_parts(name: str) -> tuple[datetime, str] | None:
+ """Parse a supported run timestamp and return its remaining suffix."""
for timestamp_format, prefix_length in _RUN_TIMESTAMP_FORMATS:
prefix = name[:prefix_length]
suffix = name[prefix_length:]
if suffix and not suffix.startswith("_"):
continue
try:
- return datetime.strptime(prefix, timestamp_format) # noqa: DTZ007 -- run names have no timezone
+ timestamp = datetime.strptime(prefix, timestamp_format) # noqa: DTZ007 -- run names have no timezone
except ValueError:
continue
+ return timestamp, suffix
return None
+def _run_timestamp(name: str) -> datetime | None:
+ """Parse the timestamp prefix shared by current and legacy run names."""
+ parsed = _run_timestamp_parts(name)
+ return parsed[0] if parsed is not None else None
+
+
def _path_is_link_or_reparse(path: Path, metadata: os.stat_result) -> bool:
"""Return whether a result path is a symlink, junction, or reparse point."""
reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
@@ -157,27 +173,52 @@ def _is_single_link_regular(path: Path, metadata: os.stat_result) -> bool:
return not _path_is_link_or_reparse(path, metadata) and stat.S_ISREG(metadata.st_mode) and metadata.st_nlink == 1
-def _read_stable_json(
+def _read_stable_bytes(
secure_root: SecureRoot,
path: Path,
before: os.stat_result,
-) -> tuple[object, os.stat_result, bytes] | None:
- """Read one JSON artifact through the pinned run root."""
+) -> tuple[os.stat_result, bytes] | None:
+ """Read one regular artifact through the pinned run root."""
if not _is_single_link_regular(path, before):
return None
try:
relative_path = path.absolute().relative_to(secure_root.root)
raw, opened = secure_root.read_bytes(relative_path, before.st_size, expected=before)
after = path.lstat()
- payload = json.loads(raw.decode("utf-8"))
- except (UnicodeDecodeError, json.JSONDecodeError, OSError, SecurePathError, ValueError):
+ except (OSError, SecurePathError, ValueError):
return None
- if (
- not _is_single_link_regular(path, after)
- or _node_fingerprint(opened) != _node_fingerprint(before)
- or _node_fingerprint(after) != _node_fingerprint(opened)
- or len(raw) != opened.st_size
- ):
+ if not _is_single_link_regular(path, after) or len(raw) != opened.st_size:
+ return None
+ if _PATH_DESCRIPTOR_IDENTITIES_COMPARABLE:
+ if _node_fingerprint(opened) != _node_fingerprint(before) or _node_fingerprint(after) != _node_fingerprint(
+ opened
+ ):
+ return None
+ stable_metadata = opened
+ else:
+ # Windows path ``lstat`` and CRT descriptor ``fstat`` expose
+ # incompatible identity fields. ``SecureRoot._open_windows`` pins the
+ # native handle and validates the declared name before the read, so the
+ # caller completes that proof with same-family path snapshots here.
+ if _node_fingerprint(after) != _node_fingerprint(before) or len(raw) != after.st_size:
+ return None
+ stable_metadata = after
+ return stable_metadata, raw
+
+
+def _read_stable_json(
+ secure_root: SecureRoot,
+ path: Path,
+ before: os.stat_result,
+) -> tuple[object, os.stat_result, bytes] | None:
+ """Read one JSON artifact through the pinned run root."""
+ observed = _read_stable_bytes(secure_root, path, before)
+ if observed is None:
+ return None
+ opened, raw = observed
+ try:
+ payload = json.loads(raw.decode("utf-8"))
+ except (UnicodeDecodeError, json.JSONDecodeError):
return None
return payload, opened, raw
@@ -192,11 +233,156 @@ def _artifact_snapshot_is_unchanged(
current_metadata = path.lstat()
except OSError:
return False
- observed = _read_stable_json(secure_root, path, current_metadata)
+ observed = _read_stable_bytes(secure_root, path, current_metadata)
return bool(
observed is not None
- and observed[2] == expected_raw
- and _node_fingerprint(observed[1]) == _node_fingerprint(expected_metadata)
+ and observed[1] == expected_raw
+ and _node_fingerprint(observed[0]) == _node_fingerprint(expected_metadata)
+ )
+
+
+def _recorded_path_matches(recorded: object, expected: Path) -> bool:
+ """Match absolute paths canonically and relative paths by safe suffix."""
+ if not isinstance(recorded, str) or not recorded:
+ return False
+ try:
+ recorded_path = Path(recorded).expanduser()
+ expected_path = expected.expanduser().resolve(strict=False)
+ except (OSError, RuntimeError):
+ return False
+ if recorded_path.is_absolute():
+ try:
+ return recorded_path.resolve(strict=False) == expected_path
+ except (OSError, RuntimeError):
+ return False
+ if not recorded_path.parts or any(part == ".." for part in recorded_path.parts):
+ return False
+ recorded_parts = tuple(os.path.normcase(part) for part in recorded_path.parts if part != ".")
+ expected_parts = tuple(os.path.normcase(part) for part in expected_path.parts)
+ return (
+ bool(recorded_parts)
+ and len(recorded_parts) <= len(expected_parts)
+ and expected_parts[-len(recorded_parts) :] == recorded_parts
+ )
+
+
+def _is_non_bool_int(value: object, *, minimum: int = 0) -> bool:
+ return isinstance(value, int) and not isinstance(value, bool) and value >= minimum
+
+
+def _is_finite_number(value: object, *, minimum: float | None = None, maximum: float | None = None) -> bool:
+ if isinstance(value, bool) or not isinstance(value, int | float):
+ return False
+ try:
+ numeric = float(value)
+ except (OverflowError, ValueError):
+ return False
+ if not math.isfinite(numeric):
+ return False
+ return (minimum is None or numeric >= minimum) and (maximum is None or numeric <= maximum)
+
+
+def _is_string_list(value: object) -> bool:
+ return isinstance(value, list) and all(isinstance(item, str) for item in value)
+
+
+def _score_mapping_is_valid(value: object) -> bool:
+ return isinstance(value, dict) and all(
+ isinstance(metric, str) and bool(metric) and _is_finite_number(score) for metric, score in value.items()
+ )
+
+
+def _current_agent_result_is_valid(
+ candidate: Path,
+ agent: str,
+ configured: dict[object, object],
+ payload: object,
+) -> bool:
+ if not isinstance(payload, dict):
+ return False
+ model = configured.get("model")
+ source = configured.get("source")
+ model_resolution = payload.get("model_resolution")
+ if (
+ payload.get("model") != model
+ or payload.get("model_source") != source
+ or not isinstance(model_resolution, dict)
+ or model_resolution.get("model") != model
+ or model_resolution.get("source") != source
+ ):
+ return False
+ for field in ("with_skill", "without_skill", "custom_with_skill", "custom_without_skill"):
+ if not _score_mapping_is_valid(payload.get(field)):
+ return False
+ for field in (
+ "dimensions_with_skill",
+ "dimensions_without_skill",
+ "lift",
+ "custom_lift",
+ "security_attribution",
+ "conditions",
+ ):
+ if not isinstance(payload.get(field), dict):
+ return False
+ pass_at_k = payload.get("pass_at_k")
+ if not isinstance(pass_at_k, dict) or any(
+ not isinstance(pass_at_k.get(condition), dict) for condition in ("with_skill", "without_skill", "lift")
+ ):
+ return False
+ for field in ("agent_runtime_failures", "trial_failures"):
+ failures = payload.get(field)
+ if not isinstance(failures, dict) or any(
+ not isinstance(failures.get(condition), list) for condition in ("with_skill", "without_skill")
+ ):
+ return False
+ job_failures = payload.get("job_failures")
+ if not isinstance(job_failures, dict) or any(
+ not isinstance(job_failures.get(condition), str) for condition in ("with_skill", "without_skill")
+ ):
+ return False
+ if payload.get("execution_status") not in _CURRENT_EXECUTION_STATUSES or not _is_string_list(
+ payload.get("execution_errors")
+ ):
+ return False
+ if any(
+ not _is_non_bool_int(payload.get(field))
+ for field in ("expected_attempts", "scored_attempts", "num_trials_with", "num_trials_without")
+ ):
+ return False
+ return _recorded_path_matches(payload.get("output_dir"), candidate / agent)
+
+
+def _current_result_identity_is_valid(candidate: Path, run_config: dict[object, object], result: object) -> bool:
+ """Validate identities emitted together for an authenticated current run."""
+ if not isinstance(result, dict) or result.get("run_id") != candidate.name or result.get("run_config") != run_config:
+ return False
+ configured_agents = _current_run_config_agents(run_config)
+ result_agents = result.get("agents")
+ if configured_agents is None or not isinstance(result_agents, dict) or set(result_agents) != set(configured_agents):
+ return False
+ if not all(
+ _current_agent_result_is_valid(candidate, agent, configured_agents[agent], result_agents[agent])
+ for agent in configured_agents
+ ):
+ return False
+ attempt_policy = result.get("attempt_policy")
+ if (
+ not isinstance(result.get("skill_name"), str)
+ or not result["skill_name"]
+ or result.get("execution_status") not in _CURRENT_EXECUTION_STATUSES
+ or not _is_string_list(result.get("execution_errors"))
+ or result.get("report_status") not in {"complete", "degraded"}
+ or not _is_finite_number(result.get("duration_seconds"), minimum=0)
+ or not isinstance(attempt_policy, dict)
+ or not _is_non_bool_int(attempt_policy.get("max_attempts"), minimum=1)
+ or not _is_finite_number(attempt_policy.get("pass_threshold"), minimum=0, maximum=1)
+ or not isinstance(attempt_policy.get("stop_on_pass"), bool)
+ or not isinstance(attempt_policy.get("score_definition"), str)
+ or not attempt_policy["score_definition"]
+ ):
+ return False
+ return _recorded_path_matches(result.get("run_dir"), candidate) and _recorded_path_matches(
+ result.get("result_path"), candidate / _FINAL_RESULT_ARTIFACT
)
@@ -240,13 +426,53 @@ def _legacy_run_agents(payload: object) -> tuple[str, ...] | None:
return tuple(sorted(normalized))
-def _is_finite_number(value: object) -> bool:
- if isinstance(value, bool) or not isinstance(value, int | float):
- return False
- try:
- return math.isfinite(value)
- except OverflowError:
- return False
+def _current_run_config_agents(payload: object) -> dict[str, dict[object, object]] | None:
+ """Return agent metadata only for the complete current runner config schema."""
+ expected_agents = _legacy_run_agents(payload)
+ if expected_agents is None or not isinstance(payload, dict):
+ return None
+ config_file = payload.get("config_file")
+ harbor = payload.get("harbor")
+ provider = payload.get("provider")
+ grading = payload.get("grading")
+ agents = payload.get("agents")
+ if (
+ not isinstance(config_file, str)
+ or not config_file
+ or not isinstance(harbor, dict)
+ or not isinstance(provider, dict)
+ or not isinstance(grading, dict)
+ or not isinstance(agents, dict)
+ ):
+ return None
+ environment = harbor.get("environment")
+ if (
+ not isinstance(environment, dict)
+ or not isinstance(environment.get("value"), str)
+ or not environment["value"]
+ or not isinstance(environment.get("source"), str)
+ or not environment["source"]
+ or not _is_non_bool_int(harbor.get("n_attempts"), minimum=1)
+ or not isinstance(harbor.get("stop_on_pass"), bool)
+ or not _is_non_bool_int(harbor.get("n_concurrent"), minimum=1)
+ or not _is_finite_number(harbor.get("timeout_multiplier"), minimum=0)
+ or harbor.get("timeout_multiplier") == 0
+ or harbor.get("base_image_mode") not in _CURRENT_BASE_IMAGE_MODES
+ or not isinstance(harbor.get("jobs_retained"), bool)
+ or not isinstance(provider.get("name"), str)
+ or not provider["name"]
+ or not isinstance(provider.get("model"), str)
+ or not provider["model"]
+ or grading.get("mode") not in _CURRENT_GRADING_MODES
+ ):
+ return None
+ normalized: dict[str, dict[object, object]] = {}
+ for agent in expected_agents:
+ metadata = agents.get(agent)
+ if not isinstance(metadata, dict) or metadata.get("agent") != agent:
+ return None
+ normalized[agent] = metadata
+ return normalized
def _is_nonnegative_int(value: object) -> bool:
@@ -591,11 +817,11 @@ def run_directory_sort_key(
) -> tuple[datetime, int, str] | None:
"""Return shared ordering metadata for one result directory.
- Timestamped runs are accepted when they carry either a valid final result
- matching the directory's run ID or the stricter pre-result legacy contract,
- and are ordered by parsed timestamp, completion mtime, then name. Compare
- also keeps accepting summary-only non-timestamp directories, placing them
- behind timestamped runs.
+ Exact-timestamp historical runs retain the prior result or pre-result
+ summary contracts. Collision-safe suffixed runs require their path-bound
+ generated-output marker plus matching result/config identities. Accepted
+ runs are ordered by parsed timestamp, completion mtime, then name. Compare
+ also keeps summary-only non-timestamp directories behind timestamped runs.
"""
if candidate.name.startswith((".", "_")) or candidate.name == "latest":
return None
@@ -606,15 +832,27 @@ def run_directory_sort_key(
except OSError:
return None
- timestamp = _run_timestamp(candidate.name)
- if timestamp is None:
+ parsed_run_name = _run_timestamp_parts(candidate.name)
+ if parsed_run_name is None:
if require_completed_result:
return None
return datetime.min, -1, candidate.name # noqa: DTZ901 -- legacy names have no timezone
+ timestamp, suffix = parsed_run_name
+ is_current_run = bool(suffix)
+ if is_current_run and _CURRENT_RUN_SUFFIX.fullmatch(suffix) is None:
+ return None
try:
snapshots: list[tuple[Path, bytes, os.stat_result]] = []
with SecureRoot(candidate, expected=candidate_metadata) as secure_root:
+ if is_current_run:
+ marker_path = candidate / GENERATED_OUTPUT_MARKER
+ marker_metadata = marker_path.lstat()
+ observed_marker = _read_stable_bytes(secure_root, marker_path, marker_metadata)
+ if observed_marker is None or not is_generated_output_root(candidate):
+ return None
+ snapshots.append((marker_path, observed_marker[1], observed_marker[0]))
+
run_config_path = candidate / _RUN_COMPLETION_ARTIFACTS[0]
run_config_metadata = run_config_path.lstat()
observed_run_config = _read_stable_json(secure_root, run_config_path, run_config_metadata)
@@ -643,7 +881,10 @@ def run_directory_sort_key(
if observed_result is None:
return None
result = observed_result[0]
- if not isinstance(result, dict) or result.get("run_id") != candidate.name:
+ if is_current_run:
+ if not _current_result_identity_is_valid(candidate, observed_run_config[0], result):
+ return None
+ elif not isinstance(result, dict) or result.get("run_id") != candidate.name:
return None
snapshots.append((result_path, observed_result[2], observed_result[1]))
completion_mtime = observed_result[1].st_mtime_ns
@@ -656,7 +897,7 @@ def run_directory_sort_key(
candidate_after
) != _node_fingerprint(candidate_metadata):
return None
- except (OSError, SecurePathError):
+ except (OSError, SecurePathError, ValueError):
return None
return timestamp, completion_mtime, candidate.name
diff --git a/src/skillevaluator/utils/secure_fs.py b/src/skillevaluator/utils/secure_fs.py
index 870039a1..587e5569 100644
--- a/src/skillevaluator/utils/secure_fs.py
+++ b/src/skillevaluator/utils/secure_fs.py
@@ -259,7 +259,12 @@ class SecureRoot:
try:
opened = os.fstat(descriptor)
- _validate_opened_file(opened, relative_path, expected)
+ # Windows path stat and CRT descriptor stat do not expose a
+ # reliably comparable st_dev/st_ino pair. _open_windows rechecks
+ # the declared name with path lstat around its native no-follow
+ # handle open; only POSIX compares the descriptor to discovery
+ # metadata here.
+ _validate_opened_file(opened, relative_path, expected if os.name == "posix" else None)
if opened.st_size > max_bytes:
raise SecurePathError(
"file_size_limit",
@@ -361,6 +366,7 @@ class SecureRoot:
directory_handles: list[int] = []
parent_handle = self._windows_root_handles[-1]
+ declared_path = self.root / relative_path
descriptor = -1
native_file_handle = -1
try:
@@ -382,6 +388,20 @@ class SecureRoot:
directory_handles.append(native_directory_handle)
parent_handle = native_directory_handle
+ # Revalidate with the same path-stat family used during discovery,
+ # then pin that name through a native handle which denies write and
+ # delete sharing. CRT fstat identity is not comparable to lstat on
+ # Windows, so name stability is established before conversion.
+ try:
+ before_open = declared_path.lstat()
+ except OSError as exc:
+ raise SecurePathError(
+ "unsafe_path",
+ f"Cannot inspect selected file securely: {relative_path.as_posix()}: {exc}",
+ relative_path=relative_path.as_posix(),
+ ) from exc
+ _validate_opened_file(before_open, relative_path, expected)
+
native_file_handle = _windows_open_relative_handle(
parent_handle,
relative_path.name,
@@ -395,13 +415,22 @@ class SecureRoot:
create_options=_WINDOWS_FILE_OPEN_OPTIONS,
)
_validate_windows_read_file_handle(native_file_handle, relative_path)
+ try:
+ after_open = declared_path.lstat()
+ except OSError as exc:
+ raise SecurePathError(
+ "unsafe_path",
+ f"Cannot revalidate selected file securely: {relative_path.as_posix()}: {exc}",
+ relative_path=relative_path.as_posix(),
+ ) from exc
+ _validate_opened_file(after_open, relative_path, before_open)
descriptor = msvcrt.open_osfhandle(
native_file_handle,
os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOINHERIT", 0),
)
native_file_handle = -1
opened = os.fstat(descriptor)
- _validate_opened_file(opened, relative_path, expected)
+ _validate_opened_file(opened, relative_path, None)
return descriptor
except OSError as exc:
if descriptor >= 0:
diff --git a/tests/test_harbor_output_provenance.py b/tests/test_harbor_output_provenance.py
index 018faa5d..a582427f 100644
--- a/tests/test_harbor_output_provenance.py
+++ b/tests/test_harbor_output_provenance.py
@@ -11,6 +11,7 @@ import shutil
import threading
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
+from types import SimpleNamespace
import pytest
@@ -57,6 +58,170 @@ def _in_skill_output(skill: Path, name: str) -> tuple[Path, Path]:
return declared_root, declared_root / "dataset"
+def _simulate_windows_crt_fstat_identity(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Return valid descriptor metadata with a Windows-incompatible identity."""
+ original_fstat = output_provenance.os.fstat
+
+ def incompatible_fstat(descriptor: int) -> object:
+ opened = original_fstat(descriptor)
+ return SimpleNamespace(
+ st_dev=opened.st_dev + 10_000,
+ st_ino=opened.st_ino + 10_000,
+ st_mode=opened.st_mode,
+ st_nlink=opened.st_nlink,
+ st_size=opened.st_size,
+ st_uid=opened.st_uid,
+ st_mtime_ns=opened.st_mtime_ns,
+ st_ctime_ns=opened.st_ctime_ns,
+ )
+
+ monkeypatch.setattr(output_provenance, "_PATH_DESCRIPTOR_IDENTITIES_COMPARABLE", False, raising=False)
+ monkeypatch.setattr(output_provenance.os, "fstat", incompatible_fstat)
+
+
+def test_atomic_output_accepts_windows_crt_descriptor_identity(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ destination = tmp_path / "result.json"
+ destination.write_bytes(b"old")
+ _simulate_windows_crt_fstat_identity(monkeypatch)
+
+ output_provenance.write_output_file_atomically(destination, b"complete")
+
+ assert destination.read_bytes() == b"complete"
+ assert not list(tmp_path.glob(".result.json.*.tmp"))
+
+
+def test_marker_failure_cleanup_accepts_windows_crt_descriptor_identity(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ root = tmp_path / "generated"
+ root.mkdir()
+ original_fdopen = output_provenance.os.fdopen
+
+ class _FailingWriter:
+ def __init__(self, handle: object) -> None:
+ self._handle = handle
+
+ def __enter__(self) -> _FailingWriter:
+ return self
+
+ def __exit__(self, *_args: object) -> None:
+ self._handle.close() # type: ignore[attr-defined]
+
+ def write(self, payload: bytes) -> int:
+ self._handle.write(payload[:1]) # type: ignore[attr-defined]
+ self._handle.flush() # type: ignore[attr-defined]
+ raise OSError("injected marker write failure")
+
+ def flush(self) -> None:
+ self._handle.flush() # type: ignore[attr-defined]
+
+ def fileno(self) -> int:
+ return self._handle.fileno() # type: ignore[attr-defined,no-any-return]
+
+ monkeypatch.setattr(output_provenance, "_load_or_create_key", lambda: b"k" * 32)
+ monkeypatch.setattr(
+ output_provenance.os,
+ "fdopen",
+ lambda descriptor, *args, **kwargs: _FailingWriter(original_fdopen(descriptor, *args, **kwargs)),
+ )
+ _simulate_windows_crt_fstat_identity(monkeypatch)
+
+ with pytest.raises(OSError, match="injected marker write failure"):
+ output_provenance.write_generated_output_marker(root)
+
+ assert list(root.iterdir()) == []
+
+
[email protected]("failure_point", ["fstat", "lstat"])
+def test_marker_metadata_failure_closes_descriptor_without_unproven_name_cleanup(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ failure_point: str,
+) -> None:
+ root = tmp_path / "generated"
+ root.mkdir()
+ marker = root / GENERATED_OUTPUT_MARKER
+ original_close = output_provenance.os.close
+ original_fstat = output_provenance.os.fstat
+ original_lstat = output_provenance.Path.lstat
+ original_unlink_if_same_file = output_provenance._unlink_if_same_file
+ closed_descriptors: list[int] = []
+ cleanup_attempts: list[Path] = []
+ failure_injected = False
+
+ def tracked_close(descriptor: int) -> None:
+ closed_descriptors.append(descriptor)
+ original_close(descriptor)
+
+ def injected_fstat(descriptor: int) -> os.stat_result:
+ nonlocal failure_injected
+ if failure_point == "fstat" and not failure_injected:
+ failure_injected = True
+ raise OSError("injected marker fstat failure")
+ return original_fstat(descriptor)
+
+ def injected_lstat(path: Path) -> os.stat_result:
+ nonlocal failure_injected
+ if failure_point == "lstat" and path == marker and not failure_injected:
+ failure_injected = True
+ raise OSError("injected marker lstat failure")
+ return original_lstat(path)
+
+ def track_cleanup(path: Path, expected: os.stat_result) -> bool:
+ cleanup_attempts.append(path)
+ return original_unlink_if_same_file(path, expected)
+
+ monkeypatch.setattr(output_provenance, "_load_or_create_key", lambda: b"k" * 32)
+ monkeypatch.setattr(output_provenance.os, "close", tracked_close)
+ monkeypatch.setattr(output_provenance.os, "fstat", injected_fstat)
+ monkeypatch.setattr(output_provenance.Path, "lstat", injected_lstat)
+ monkeypatch.setattr(output_provenance, "_unlink_if_same_file", track_cleanup)
+
+ with pytest.raises(OSError, match=rf"injected marker {failure_point} failure"):
+ output_provenance.write_generated_output_marker(root)
+
+ assert failure_injected
+ assert closed_descriptors
+ assert cleanup_attempts == []
+ assert marker.is_file()
+
+
[email protected](os.name != "posix", reason="requires POSIX rename-while-open behavior")
+def test_marker_lstat_failure_never_unlinks_a_substituted_name(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ root = tmp_path / "generated"
+ root.mkdir()
+ marker = root / GENERATED_OUTPUT_MARKER
+ original_partial = root / ".original-partial-marker"
+ original_lstat = output_provenance.Path.lstat
+ substituted = False
+
+ def substitute_then_fail(path: Path) -> os.stat_result:
+ nonlocal substituted
+ if path == marker and not substituted:
+ substituted = True
+ marker.rename(original_partial)
+ marker.write_text("preserve replacement\n", encoding="utf-8")
+ raise OSError("injected marker lstat failure after substitution")
+ return original_lstat(path)
+
+ monkeypatch.setattr(output_provenance, "_load_or_create_key", lambda: b"k" * 32)
+ monkeypatch.setattr(output_provenance.Path, "lstat", substitute_then_fail)
+
+ with pytest.raises(OSError, match="injected marker lstat failure after substitution"):
+ output_provenance.write_generated_output_marker(root)
+
+ assert substituted
+ assert marker.read_text(encoding="utf-8") == "preserve replacement\n"
+ assert original_partial.is_file()
+
+
def test_concurrent_first_key_creation_publishes_one_complete_key(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
@@ -101,6 +266,113 @@ def test_interrupted_hardlink_publish_is_recovered(tmp_path: Path) -> None:
assert key_path.stat().st_nlink == 1
[email protected]("failure_point", ["write", "fsync"])
+def test_marker_publication_failure_removes_only_its_partial_file(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ failure_point: str,
+) -> None:
+ """A failed marker claim must not leave a plausible owned reservation."""
+ root = tmp_path / "generated"
+ root.mkdir()
+ output_provenance.generated_output_marker_payload(tmp_path / "seed")
+ original_fdopen = output_provenance.os.fdopen
+ original_fsync = output_provenance.os.fsync
+
+ class _PartialWriter:
+ def __init__(self, handle: object) -> None:
+ self._handle = handle
+
+ def __enter__(self) -> _PartialWriter:
+ return self
+
+ def __exit__(self, *_args: object) -> None:
+ self._handle.close() # type: ignore[attr-defined]
+
+ def write(self, payload: bytes) -> int:
+ self._handle.write(payload[: len(payload) // 2]) # type: ignore[attr-defined]
+ self._handle.flush() # type: ignore[attr-defined]
+ raise OSError("injected marker write failure")
+
+ def flush(self) -> None:
+ self._handle.flush() # type: ignore[attr-defined]
+
+ def fileno(self) -> int:
+ return self._handle.fileno() # type: ignore[attr-defined,no-any-return]
+
+ if failure_point == "write":
+ monkeypatch.setattr(
+ output_provenance.os,
+ "fdopen",
+ lambda descriptor, *args, **kwargs: _PartialWriter(original_fdopen(descriptor, *args, **kwargs)),
+ )
+ expected_error = "injected marker write failure"
+ else:
+ monkeypatch.setattr(
+ output_provenance.os,
+ "fsync",
+ lambda _descriptor: (_ for _ in ()).throw(OSError("injected marker fsync failure")),
+ )
+ expected_error = "injected marker fsync failure"
+
+ with pytest.raises(OSError, match=expected_error):
+ output_provenance.write_generated_output_marker(root)
+
+ assert not (root / GENERATED_OUTPUT_MARKER).exists()
+ assert list(root.iterdir()) == []
+ monkeypatch.setattr(output_provenance.os, "fdopen", original_fdopen)
+ monkeypatch.setattr(output_provenance.os, "fsync", original_fsync)
+
+
+def test_owned_output_cleanup_refuses_a_path_substituted_after_authentication(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ root = tmp_path / "reserved-run"
+ moved_root = tmp_path / "moved-reserved-run"
+ mark_generated_output_root(root)
+ original_identity = root.stat().st_dev, root.stat().st_ino
+ original_fingerprint = output_provenance._node_fingerprint
+ root_fingerprint_calls = 0
+
+ def substitute_after_identity_check(metadata: os.stat_result) -> tuple[int, int, int, int, int]:
+ nonlocal root_fingerprint_calls
+ fingerprint = original_fingerprint(metadata)
+ if (metadata.st_dev, metadata.st_ino) == original_identity:
+ root_fingerprint_calls += 1
+ if root_fingerprint_calls == 2:
+ root.rename(moved_root)
+ root.mkdir()
+ (root / "preserve.txt").write_text("replacement\n", encoding="utf-8")
+ return fingerprint
+
+ monkeypatch.setattr(output_provenance, "_node_fingerprint", substitute_after_identity_check)
+
+ removed = output_provenance.remove_generated_output_root_if_owned(root)
+
+ assert removed is False
+ assert (root / "preserve.txt").read_text(encoding="utf-8") == "replacement\n"
+ assert moved_root.is_dir()
+
+
+def test_fallback_owned_output_cleanup_never_recursively_deletes_contents(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ root = tmp_path / "reserved-run"
+ mark_generated_output_root(root)
+ sentinel = root / "preserve.txt"
+ sentinel.write_text("preserve\n", encoding="utf-8")
+ identity = root.stat().st_dev, root.stat().st_ino
+ monkeypatch.setattr(output_provenance, "_DESCRIPTOR_BACKEND", False)
+
+ removed = output_provenance.remove_generated_output_root_if_owned(root, expected_identity=identity)
+
+ assert removed is False
+ assert sentinel.read_text(encoding="utf-8") == "preserve\n"
+ assert (root / GENERATED_OUTPUT_MARKER).is_file()
+
+
def test_fixed_marker_cannot_authorize_authored_source_replacement(tmp_path: Path) -> None:
skill = _write_skill(tmp_path)
declared_root, output = _in_skill_output(skill, "fixed-marker-results")
diff --git a/tests/test_harbor_runtime_skill_isolation.py b/tests/test_harbor_runtime_skill_isolation.py
index 48685da3..baa59e03 100644
--- a/tests/test_harbor_runtime_skill_isolation.py
+++ b/tests/test_harbor_runtime_skill_isolation.py
@@ -8,10 +8,12 @@ from __future__ import annotations
import json
import os
import shutil
+import stat
import subprocess
import sys
import tempfile
from pathlib import Path
+from types import SimpleNamespace
import pytest
@@ -722,15 +724,21 @@ def test_prebuild_uses_each_tasks_exact_environment(
assert task_tags["case-a"] != task_tags["case-b"]
first_build_tags = {path.parent.name: tag for tag, path in build_calls}
- (dataset / "case-a" / "environment" / "input" / "fixture.txt").chmod(0o755)
- for task in dataset.iterdir():
- (task / "task.toml").write_text("[environment]\n", encoding="utf-8")
- build_calls.clear()
-
- assert prebuild_task_environments([dataset]) == 2
- second_build_tags = {path.parent.name: tag for tag, path in build_calls}
- assert second_build_tags["case-a"] != first_build_tags["case-a"]
- assert second_build_tags["case-b"] == first_build_tags["case-b"]
+ fixture_path = dataset / "case-a" / "environment" / "input" / "fixture.txt"
+ original_mode = stat.S_IMODE(fixture_path.stat().st_mode)
+ changed_mode = original_mode & ~stat.S_IWRITE if os.name == "nt" else original_mode | stat.S_IXUSR
+ try:
+ fixture_path.chmod(changed_mode)
+ for task in dataset.iterdir():
+ (task / "task.toml").write_text("[environment]\n", encoding="utf-8")
+ build_calls.clear()
+
+ assert prebuild_task_environments([dataset]) == 2
+ second_build_tags = {path.parent.name: tag for tag, path in build_calls}
+ assert second_build_tags["case-a"] != first_build_tags["case-a"]
+ assert second_build_tags["case-b"] == first_build_tags["case-b"]
+ finally:
+ fixture_path.chmod(original_mode)
def test_prebuild_skips_task_with_compose_build_override(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
@@ -3041,6 +3049,193 @@ def test_selective_evaluator_snapshot_rejects_fixture_selection_change_during_co
assert mutated
+ ("task_source", "selected_relative"),
+ [
+ ("evals_json", Path("files/selected.txt")),
+ ("evals_json", Path("grader.py")),
+ ("evals_json", Path("environment/sidecar/seed.txt")),
+ ("native_harbor", Path("harbor/case-001/instruction.md")),
+ ],
+)
+def test_selective_evaluator_snapshot_rejects_selected_content_change_during_copy(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ task_source: str,
+ selected_relative: Path,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ if task_source == "native_harbor":
+ _write_minimal_native_task(target)
+ selected_path = target / "evals" / selected_relative
+ original_copytree = adapter_module.copytree_secure
+ mutated = False
+
+ def mutate_content_then_copy(source: Path, destination: Path, *args: object, **kwargs: object) -> None:
+ nonlocal mutated
+ if Path(source) == target / "evals" and not mutated:
+ selected_path.write_text(selected_path.read_text(encoding="utf-8") + "MUTATED\n", encoding="utf-8")
+ mutated = True
+ original_copytree(source, destination, *args, **kwargs)
+
+ monkeypatch.setattr(adapter_module, "copytree_secure", mutate_content_then_copy)
+
+ with (
+ pytest.raises(ValueError, match="selection changed"),
+ private_evaluator_skill_snapshot(target, task_source=task_source),
+ ):
+ pass
+
+ assert mutated
+
+
[email protected](os.name != "posix", reason="requires POSIX executable mode semantics")
+def test_selective_evaluator_snapshot_rejects_selected_mode_change_during_copy(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ script = target / "evals" / "environment" / "run.sh"
+ script.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
+ script.chmod(0o644)
+ original_copytree = adapter_module.copytree_secure
+ mutated = False
+
+ def mutate_mode_then_copy(source: Path, destination: Path, *args: object, **kwargs: object) -> None:
+ nonlocal mutated
+ if Path(source) == target / "evals" and not mutated:
+ script.chmod(0o755)
+ mutated = True
+ original_copytree(source, destination, *args, **kwargs)
+
+ monkeypatch.setattr(adapter_module, "copytree_secure", mutate_mode_then_copy)
+
+ with (
+ pytest.raises(ValueError, match="selection changed"),
+ private_evaluator_skill_snapshot(target, task_source="evals_json"),
+ ):
+ pass
+
+ assert mutated
+
+
+def test_evaluator_read_accepts_windows_crt_descriptor_identity(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ evals = tmp_path / "evals"
+ evals.mkdir()
+ dataset = evals / "evals.json"
+ dataset.write_text("[]\n", encoding="utf-8")
+ original_read_bytes = adapter_module.SecureRoot.read_bytes
+
+ def read_with_windows_crt_identity(
+ secure_root: adapter_module.SecureRoot,
+ relative_path: Path,
+ max_bytes: int,
+ *,
+ expected: os.stat_result | None = None,
+ ) -> tuple[bytes, object]:
+ raw, opened = original_read_bytes(secure_root, relative_path, max_bytes, expected=expected)
+ return raw, SimpleNamespace(
+ st_dev=opened.st_dev + 10_000,
+ st_ino=opened.st_ino + 10_000,
+ st_mode=opened.st_mode,
+ st_nlink=opened.st_nlink,
+ st_size=opened.st_size,
+ st_mtime_ns=opened.st_mtime_ns,
+ st_ctime_ns=opened.st_ctime_ns,
+ )
+
+ monkeypatch.setattr(adapter_module, "_PATH_DESCRIPTOR_IDENTITIES_COMPARABLE", False, raising=False)
+ monkeypatch.setattr(adapter_module.SecureRoot, "read_bytes", read_with_windows_crt_identity)
+
+ assert adapter_module._read_regular_evals_file(dataset, allowed_root=evals) == b"[]\n"
+
+
[email protected]("task_source", ["evals_json", "native_harbor"])
+def test_baseline_alias_candidates_are_scanned_once_per_multi_case_staging(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ task_source: str,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ entries = [
+ {
+ "id": f"case-{index:03d}",
+ "question": f"Run case {index}.",
+ "files": ["evals/files/selected.txt"],
+ }
+ for index in range(1, 4)
+ ]
+ (target / "evals" / "evals.json").write_text(json.dumps(entries), encoding="utf-8")
+ if task_source == "native_harbor":
+ for entry in entries:
+ native_task = target / "evals" / "harbor" / entry["id"]
+ native_task.mkdir(parents=True)
+ (native_task / "instruction.md").write_text("Run the native case.\n", encoding="utf-8")
+ (native_task / "task.toml").write_text(
+ 'schema_version = "1.3"\n\n[task]\n'
+ f'name = "nvidia/{entry["id"]}"\n\n'
+ f'[metadata]\nentry_id = "{entry["id"]}"\n\n[environment]\n',
+ encoding="utf-8",
+ )
+
+ scans = 0
+
+ def count_scan(*args: object, **kwargs: object) -> None:
+ nonlocal scans
+ scans += 1
+
+ monkeypatch.setattr(adapter_module, "_check_baseline_skill_candidates_do_not_alias_target", count_scan)
+ stager = generate_harbor_tasks if task_source == "evals_json" else stage_native_harbor_tasks
+
+ tasks = stager(target, tmp_path / f"{task_source}-scan-count", with_skill=False)
+
+ assert len(tasks) == 3
+ assert scans == 1
+
+
+def test_run_scoped_baseline_alias_validation_skips_rescan_and_is_path_bound(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _, target, references_dir, workspace = _write_projection_fixture(tmp_path)
+ excluded_root = target / "evals" / "results"
+ validation = adapter_module._prevalidate_baseline_skill_candidates(
+ target,
+ references_dir,
+ [workspace],
+ excluded_roots=(excluded_root,),
+ )
+
+ def reject_rescan(*args: object, **kwargs: object) -> None:
+ raise AssertionError("run-scoped validation should suppress repeated source scans")
+
+ monkeypatch.setattr(adapter_module, "_check_baseline_skill_candidates_do_not_alias_target", reject_rescan)
+ tasks = generate_harbor_tasks(
+ target,
+ tmp_path / "validated-baseline",
+ with_skill=False,
+ reference_skills_dir=references_dir,
+ workspace_skill_paths=[workspace],
+ repo_context_exclude_paths=(excluded_root,),
+ _baseline_alias_validation=validation,
+ )
+
+ assert len(tasks) == 1
+ with pytest.raises(ValueError, match="does not match"):
+ generate_harbor_tasks(
+ target,
+ tmp_path / "mismatched-validation",
+ with_skill=False,
+ reference_skills_dir=references_dir,
+ workspace_skill_paths=[],
+ repo_context_exclude_paths=(excluded_root,),
+ _baseline_alias_validation=validation,
+ )
+
+
def test_in_repo_temp_evaluator_snapshot_is_excluded_from_copy_repo(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/test_harbor_secure_copy.py b/tests/test_harbor_secure_copy.py
index ba25ba57..305561fe 100644
--- a/tests/test_harbor_secure_copy.py
+++ b/tests/test_harbor_secure_copy.py
@@ -17,6 +17,7 @@ import time
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import TimeoutError as FutureTimeoutError
from pathlib import Path
+from types import SimpleNamespace
import pytest
@@ -35,6 +36,15 @@ def _copy_file_secure(source: Path, destination: Path, **kwargs: object) -> None
module.copy_file_secure(source, destination, **kwargs)
+def _assert_mode_preserved(source: Path, destination: Path) -> None:
+ source_mode = stat.S_IMODE(source.stat().st_mode)
+ destination_mode = stat.S_IMODE(destination.stat().st_mode)
+ if os.name == "nt":
+ assert bool(destination_mode & stat.S_IWRITE) == bool(source_mode & stat.S_IWRITE)
+ else:
+ assert destination_mode == source_mode
+
+
def test_public_secure_copy_api_documents_same_uid_boundary() -> None:
module = importlib.import_module("skillevaluator.tier3.harbor.secure_copy")
@@ -595,7 +605,7 @@ def test_regular_copy_preserves_modes_and_merges_transactionally(tmp_path: Path)
assert (destination / "unrelated.txt").read_text(encoding="utf-8") == "keep"
assert (destination / "nested" / "run.sh").read_text(encoding="utf-8") == "#!/bin/sh\n"
- assert stat.S_IMODE((destination / "nested" / "run.sh").stat().st_mode) == 0o751
+ _assert_mode_preserved(script, destination / "nested" / "run.sh")
def test_private_stage_injection_is_rejected_before_publish(
@@ -651,7 +661,92 @@ def test_checked_non_posix_fallback_copies_tree_and_file(
file_destination.write_text("old", encoding="utf-8")
module.copy_file_secure(file_source, file_destination, allowed_root=tmp_path)
assert file_destination.read_text(encoding="utf-8") == "new"
- assert stat.S_IMODE(file_destination.stat().st_mode) == 0o640
+ _assert_mode_preserved(file_source, file_destination)
+
+
+def test_checked_windows_fallback_accepts_crt_descriptor_identity(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ module = importlib.import_module("skillevaluator.tier3.harbor.secure_copy")
+ original_fstat = module.os.fstat
+
+ def windows_crt_fstat(descriptor: int) -> object:
+ opened = original_fstat(descriptor)
+ return SimpleNamespace(
+ st_dev=opened.st_dev + 10_000,
+ st_ino=opened.st_ino + 10_000,
+ st_mode=opened.st_mode,
+ st_nlink=opened.st_nlink,
+ st_size=opened.st_size,
+ st_mtime_ns=opened.st_mtime_ns,
+ st_ctime_ns=opened.st_ctime_ns,
+ )
+
+ monkeypatch.setattr(module, "_DESCRIPTOR_BACKEND", False)
+ monkeypatch.setattr(module, "_ATOMIC_RENAME", None)
+ monkeypatch.setattr(module, "_PATH_DESCRIPTOR_IDENTITIES_COMPARABLE", False, raising=False)
+ monkeypatch.setattr(module.os, "fstat", windows_crt_fstat)
+ source = tmp_path / "source"
+ source.mkdir()
+ (source / "safe.txt").write_text("tree", encoding="utf-8")
+ tree_destination = tmp_path / "tree-destination"
+ file_source = tmp_path / "file-source.txt"
+ file_source.write_text("file", encoding="utf-8")
+ file_destination = tmp_path / "file-destination.txt"
+
+ module.copytree_secure(source, tree_destination, allowed_root=tmp_path)
+ module.copy_file_secure(file_source, file_destination, allowed_root=tmp_path)
+
+ assert (tree_destination / "safe.txt").read_text(encoding="utf-8") == "tree"
+ assert file_destination.read_text(encoding="utf-8") == "file"
+
+
[email protected](os.name != "nt", reason="requires native Windows filesystem semantics")
+def test_native_windows_fallback_copies_tree_and_file(tmp_path: Path) -> None:
+ module = importlib.import_module("skillevaluator.tier3.harbor.secure_copy")
+ source = tmp_path / "source"
+ source.mkdir()
+ source_file = source / "writable.txt"
+ source_file.write_text("tree", encoding="utf-8")
+ source_file.chmod(0o666)
+ tree_destination = tmp_path / "tree-destination"
+ file_source = tmp_path / "read-only.txt"
+ file_source.write_text("file", encoding="utf-8")
+ file_source.chmod(0o444)
+ file_destination = tmp_path / "file-destination.txt"
+
+ try:
+ module.copytree_secure(source, tree_destination, allowed_root=tmp_path)
+ module.copy_file_secure(file_source, file_destination, allowed_root=tmp_path)
+
+ assert (tree_destination / "writable.txt").read_text(encoding="utf-8") == "tree"
+ assert (tree_destination / "writable.txt").stat().st_mode & stat.S_IWRITE
+ assert file_destination.read_text(encoding="utf-8") == "file"
+ assert not file_destination.stat().st_mode & stat.S_IWRITE
+ finally:
+ file_source.chmod(0o666)
+ if file_destination.exists():
+ file_destination.chmod(0o666)
+
+
[email protected](os.name != "nt", reason="requires native Windows chmod semantics")
+def test_native_windows_tree_fingerprint_binds_writable_mode(tmp_path: Path) -> None:
+ module = importlib.import_module("skillevaluator.tier3.harbor.secure_copy")
+ source = tmp_path / "source"
+ source.mkdir()
+ script = source / "run.cmd"
+ script.write_text("@echo off\r\n", encoding="utf-8")
+ script.chmod(0o666)
+
+ try:
+ writable = module.tree_content_fingerprint_secure(source, allowed_root=tmp_path)
+ script.chmod(0o444)
+ read_only = module.tree_content_fingerprint_secure(source, allowed_root=tmp_path)
+ finally:
+ script.chmod(0o666)
+
+ assert writable != read_only
def test_checked_non_posix_tree_fallback_without_fchmod_preserves_file_mode(
@@ -661,7 +756,7 @@ def test_checked_non_posix_tree_fallback_without_fchmod_preserves_file_mode(
module = importlib.import_module("skillevaluator.tier3.harbor.secure_copy")
monkeypatch.setattr(module, "_DESCRIPTOR_BACKEND", False)
monkeypatch.setattr(module, "_ATOMIC_RENAME", None)
- monkeypatch.delattr(module.os, "fchmod")
+ monkeypatch.delattr(module.os, "fchmod", raising=False)
source = tmp_path / "source"
source.mkdir()
source_file = source / "safe.txt"
@@ -673,7 +768,7 @@ def test_checked_non_posix_tree_fallback_without_fchmod_preserves_file_mode(
staged_file = destination / "safe.txt"
assert staged_file.read_text(encoding="utf-8") == "safe"
- assert stat.S_IMODE(staged_file.stat().st_mode) == 0o640
+ _assert_mode_preserved(source_file, staged_file)
def test_checked_non_posix_file_fallback_without_fchmod_preserves_mode(
@@ -683,7 +778,7 @@ def test_checked_non_posix_file_fallback_without_fchmod_preserves_mode(
module = importlib.import_module("skillevaluator.tier3.harbor.secure_copy")
monkeypatch.setattr(module, "_DESCRIPTOR_BACKEND", False)
monkeypatch.setattr(module, "_ATOMIC_RENAME", None)
- monkeypatch.delattr(module.os, "fchmod")
+ monkeypatch.delattr(module.os, "fchmod", raising=False)
source = tmp_path / "source.txt"
source.write_text("safe", encoding="utf-8")
source.chmod(0o640)
@@ -692,7 +787,7 @@ def test_checked_non_posix_file_fallback_without_fchmod_preserves_mode(
module.copy_file_secure(source, destination, allowed_root=tmp_path)
assert destination.read_text(encoding="utf-8") == "safe"
- assert stat.S_IMODE(destination.stat().st_mode) == 0o640
+ _assert_mode_preserved(source, destination)
def test_checked_windows_fallback_verifies_portable_chmod_semantics(
@@ -718,7 +813,7 @@ def test_checked_windows_fallback_verifies_portable_chmod_semantics(
monkeypatch.setattr(module, "_DESCRIPTOR_BACKEND", False)
monkeypatch.setattr(module, "_ATOMIC_RENAME", None)
monkeypatch.setattr(module, "_WINDOWS_CHMOD_SEMANTICS", True)
- monkeypatch.delattr(module.os, "fchmod")
+ monkeypatch.delattr(module.os, "fchmod", raising=False)
monkeypatch.setattr(module.Path, "chmod", windows_chmod)
tree_destination = tmp_path / "tree-destination"
file_destination = tmp_path / "file-destination.txt"
diff --git a/tests/test_results_location.py b/tests/test_results_location.py
index 5820596c..758b6287 100644
--- a/tests/test_results_location.py
+++ b/tests/test_results_location.py
@@ -7,12 +7,14 @@ from __future__ import annotations
import json
import os
+import shutil
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import pytest
from skillevaluator.tier3 import results_location
+from skillevaluator.tier3.output_provenance import mark_generated_output_root
from skillevaluator.tier3.results_location import external_results_root, resolve_latest_results
@@ -27,6 +29,93 @@ def _write_completed_run(root: Path, run_id: str) -> Path:
return run_dir
+def _write_authenticated_current_run(
+ root: Path,
+ run_id: str,
+ *,
+ skill_name: str = "demo",
+ recorded_run_dir: str | None = None,
+ recorded_result_path: str | None = None,
+) -> Path:
+ run_dir = root / run_id
+ run_dir.mkdir(parents=True)
+ mark_generated_output_root(run_dir)
+ run_config = {
+ "config_file": "none",
+ "harbor": {
+ "environment": {"value": "local", "source": "test"},
+ "n_attempts": 1,
+ "stop_on_pass": False,
+ "n_concurrent": 1,
+ "timeout_multiplier": 1.0,
+ "base_image_mode": "disabled",
+ "jobs_retained": False,
+ },
+ "provider": {"name": "nvidia", "model": "test-model"},
+ "task_source": "evals_json",
+ "grading": {"mode": "default"},
+ "agents": {
+ "opencode": {
+ "agent": "opencode",
+ "model": "test-model",
+ "source": "test default",
+ }
+ },
+ }
+ agent_dir = run_dir / "opencode"
+ agent_result = {
+ "model": "test-model",
+ "model_source": "test default",
+ "model_resolution": {"model": "test-model", "source": "test default"},
+ "with_skill": {"security": 0.8},
+ "without_skill": {},
+ "custom_with_skill": {},
+ "custom_without_skill": {},
+ "dimensions_with_skill": {},
+ "dimensions_without_skill": {},
+ "lift": {},
+ "custom_lift": {},
+ "pass_at_k": {"with_skill": {}, "without_skill": {}, "lift": {}},
+ "security_attribution": {},
+ "agent_runtime_failures": {"with_skill": [], "without_skill": []},
+ "trial_failures": {"with_skill": [], "without_skill": []},
+ "job_failures": {"with_skill": "", "without_skill": ""},
+ "conditions": {"with_skill": {}, "without_skill": {}},
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ "num_trials_with": 1,
+ "num_trials_without": 0,
+ "output_dir": str(agent_dir.resolve()),
+ }
+ (run_dir / "run_config.json").write_text(json.dumps(run_config), encoding="utf-8")
+ (run_dir / "result.json").write_text(
+ json.dumps(
+ {
+ "skill_name": skill_name,
+ "run_id": run_id,
+ "run_dir": recorded_run_dir or str(run_dir),
+ "result_path": recorded_result_path or str((run_dir / "result.json").resolve()),
+ "run_config": run_config,
+ "agents": {"opencode": agent_result},
+ "attempt_policy": {
+ "max_attempts": 1,
+ "pass_threshold": 0.5,
+ "stop_on_pass": False,
+ "score_definition": "test",
+ },
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "report_status": "complete",
+ "duration_seconds": 1.0,
+ }
+ ),
+ encoding="utf-8",
+ )
+ return run_dir
+
+
def test_latest_results_falls_back_to_newest_completed_run_without_symlink(tmp_path: Path) -> None:
"""Windows users can discover runs when creating ``latest`` is not permitted."""
skill_path = tmp_path / "demo"
@@ -46,21 +135,166 @@ def test_latest_results_fallback_accepts_unique_suffixed_run_ids(tmp_path: Path)
skill_path.mkdir()
cli_results_dir = tmp_path / "results"
root = external_results_root(cli_results_dir, skill_path)
- _write_completed_run(root, "20260705_120000_111_aaaaaaaaaaaa")
- newest = _write_completed_run(root, "20260705_130000_222_bbbbbbbbbbbb")
+ _write_authenticated_current_run(root, "20260705_120000_111_aaaaaaaaaaaa")
+ newest = _write_authenticated_current_run(root, "20260705_130000_222_bbbbbbbbbbbb")
resolved = resolve_latest_results(skill_path, cli_results_dir, environ={})
assert resolved == newest
+def test_latest_results_rejects_unmarked_suffixed_run_that_would_shadow_historical_run(tmp_path: Path) -> None:
+ skill_path = tmp_path / "demo"
+ skill_path.mkdir()
+ cli_results_dir = tmp_path / "results"
+ root = external_results_root(cli_results_dir, skill_path)
+ historical = _write_completed_run(root, "20260705_120000")
+ _write_completed_run(root, "20260705_130000_222_bbbbbbbbbbbb")
+
+ resolved = resolve_latest_results(skill_path, cli_results_dir, environ={})
+
+ assert resolved == historical
+
+
+def test_latest_results_rejects_current_run_copied_across_results_roots(tmp_path: Path) -> None:
+ skill_path = tmp_path / "demo"
+ skill_path.mkdir()
+ cli_results_dir = tmp_path / "primary"
+ primary_root = external_results_root(cli_results_dir, skill_path)
+ historical = _write_completed_run(skill_path / "evals" / "results", "20260705_120000")
+ authentic = _write_authenticated_current_run(
+ tmp_path / "source" / skill_path.name,
+ "20260705_130000_222_bbbbbbbbbbbb",
+ )
+ copied = shutil.copytree(authentic, primary_root / authentic.name)
+ copied_result_path = copied / "result.json"
+ copied_result = json.loads(copied_result_path.read_text(encoding="utf-8"))
+ copied_result["run_dir"] = str(copied)
+ copied_result_path.write_text(json.dumps(copied_result), encoding="utf-8")
+
+ resolved = resolve_latest_results(skill_path, cli_results_dir, environ={})
+
+ assert resolved == historical
+
+
+ "mutate_result",
+ [
+ lambda result: result["run_config"].update({"task_source": "native_harbor"}),
+ lambda result: result.update({"run_dir": "/tmp/different-run"}),
+ ],
+ ids=("run-config", "run-directory"),
+)
+def test_latest_results_rejects_authenticated_current_run_with_inconsistent_identity(
+ tmp_path: Path,
+ mutate_result,
+) -> None:
+ skill_path = tmp_path / "demo"
+ skill_path.mkdir()
+ cli_results_dir = tmp_path / "results"
+ root = external_results_root(cli_results_dir, skill_path)
+ historical = _write_completed_run(root, "20260705_120000")
+ current = _write_authenticated_current_run(root, "20260705_130000_222_bbbbbbbbbbbb")
+ result_path = current / "result.json"
+ result = json.loads(result_path.read_text(encoding="utf-8"))
+ mutate_result(result)
+ result_path.write_text(json.dumps(result), encoding="utf-8")
+
+ resolved = resolve_latest_results(skill_path, cli_results_dir, environ={})
+
+ assert resolved == historical
+
+
+ "mutate_config",
+ [
+ lambda config: config.clear(),
+ lambda config: config.update({"agents": {}}),
+ lambda config: config["agents"]["opencode"].update({"model": ""}),
+ lambda config: config["harbor"].pop("n_attempts"),
+ ],
+ ids=("empty", "empty-agents", "empty-agent-model", "missing-harbor-field"),
+)
+def test_latest_results_rejects_authenticated_current_run_with_malformed_config_schema(
+ tmp_path: Path,
+ mutate_config,
+) -> None:
+ skill_path = tmp_path / "demo"
+ skill_path.mkdir()
+ cli_results_dir = tmp_path / "results"
+ root = external_results_root(cli_results_dir, skill_path)
+ historical = _write_completed_run(root, "20260705_120000")
+ current = _write_authenticated_current_run(root, "20260705_130000_222_bbbbbbbbbbbb")
+ config_path = current / "run_config.json"
+ result_path = current / "result.json"
+ config = json.loads(config_path.read_text(encoding="utf-8"))
+ result = json.loads(result_path.read_text(encoding="utf-8"))
+ mutate_config(config)
+ result["run_config"] = config
+ config_path.write_text(json.dumps(config), encoding="utf-8")
+ result_path.write_text(json.dumps(result), encoding="utf-8")
+
+ assert resolve_latest_results(skill_path, cli_results_dir, environ={}) == historical
+
+
+ "mutate_result",
+ [
+ lambda result: result.update({"agents": {}}),
+ lambda result: result["agents"].update({"unexpected": result["agents"]["opencode"]}),
+ lambda result: result["agents"].update({"opencode": []}),
+ lambda result: result["agents"]["opencode"].update({"with_skill": []}),
+ lambda result: result["agents"]["opencode"]["model_resolution"].update({"model": "other-model"}),
+ lambda result: result.pop("execution_status"),
+ ],
+ ids=("empty", "extra-agent", "non-object-agent", "non-object-scores", "model-mismatch", "missing-status"),
+)
+def test_latest_results_rejects_authenticated_current_run_with_malformed_result_schema(
+ tmp_path: Path,
+ mutate_result,
+) -> None:
+ skill_path = tmp_path / "demo"
+ skill_path.mkdir()
+ cli_results_dir = tmp_path / "results"
+ root = external_results_root(cli_results_dir, skill_path)
+ historical = _write_completed_run(root, "20260705_120000")
+ current = _write_authenticated_current_run(root, "20260705_130000_222_bbbbbbbbbbbb")
+ result_path = current / "result.json"
+ result = json.loads(result_path.read_text(encoding="utf-8"))
+ mutate_result(result)
+ result_path.write_text(json.dumps(result), encoding="utf-8")
+
+ assert resolve_latest_results(skill_path, cli_results_dir, environ={}) == historical
+
+
+def test_latest_results_accepts_authenticated_current_run_with_relative_recorded_path_after_cwd_change(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ creation_cwd = tmp_path / "created-from"
+ creation_cwd.mkdir()
+ root = creation_cwd / "relative-results" / "demo"
+ run_id = "20260705_130000_222_bbbbbbbbbbbb"
+ current = _write_authenticated_current_run(
+ root,
+ run_id,
+ recorded_run_dir=str(Path("relative-results") / "demo" / run_id),
+ recorded_result_path=str(Path("relative-results") / "demo" / run_id / "result.json"),
+ )
+ different_cwd = tmp_path / "different-cwd"
+ different_cwd.mkdir()
+ monkeypatch.chdir(different_cwd)
+
+ assert results_location.run_directory_sort_key(current, require_completed_result=True) is not None
+
+
def test_latest_results_same_timestamp_uses_completion_mtime(tmp_path: Path) -> None:
skill_path = tmp_path / "demo"
skill_path.mkdir()
cli_results_dir = tmp_path / "results"
root = external_results_root(cli_results_dir, skill_path)
- lexically_later = _write_completed_run(root, "20260705_130000_999_ffffffffffff")
- completed_later = _write_completed_run(root, "20260705_130000_111_aaaaaaaaaaaa")
+ lexically_later = _write_authenticated_current_run(root, "20260705_130000_999_ffffffffffff")
+ completed_later = _write_authenticated_current_run(root, "20260705_130000_111_aaaaaaaaaaaa")
os.utime(lexically_later / "result.json", ns=(100, 100))
os.utime(completed_later / "result.json", ns=(200, 200))
@@ -74,8 +308,8 @@ def test_latest_results_same_completion_time_uses_deterministic_name_order(tmp_p
skill_path.mkdir()
cli_results_dir = tmp_path / "results"
root = external_results_root(cli_results_dir, skill_path)
- lexically_first = _write_completed_run(root, "20260705_130000_111_aaaaaaaaaaaa")
- lexically_last = _write_completed_run(root, "20260705_130000_999_ffffffffffff")
+ lexically_first = _write_authenticated_current_run(root, "20260705_130000_111_aaaaaaaaaaaa")
+ lexically_last = _write_authenticated_current_run(root, "20260705_130000_999_ffffffffffff")
os.utime(lexically_first / "result.json", ns=(100, 100))
os.utime(lexically_last / "result.json", ns=(100, 100))
diff --git a/tests/test_results_location_legacy_compatibility.py b/tests/test_results_location_legacy_compatibility.py
index faa4a28f..033a860b 100644
--- a/tests/test_results_location_legacy_compatibility.py
+++ b/tests/test_results_location_legacy_compatibility.py
@@ -9,15 +9,16 @@ import json
import os
from collections.abc import Iterator
from pathlib import Path
+from types import SimpleNamespace
import pytest
-from skillevaluator.tier3 import results_location
+from skillevaluator.tier3 import output_provenance, results_location
from skillevaluator.tier3.harbor.metrics import DEFAULT_METRIC_SET
from skillevaluator.tier3.output_provenance import GENERATED_OUTPUT_MARKER, mark_generated_output_root
-def _run(root: Path, name: str = "20260709_120000_1_aaaaaa") -> Path:
+def _run(root: Path, name: str = "20260709_120000") -> Path:
candidate = root / name
candidate.mkdir(parents=True)
return candidate
@@ -108,10 +109,75 @@ def _write_successful_summary(run: Path, *, agent: str = "opencode") -> Path:
return summary
-def _write_current_run(root: Path, name: str = "20260709_120000_1_aaaaaa") -> Path:
+def _write_current_run(root: Path, name: str = "20260709_120000_1_aaaaaaaaaaaa") -> Path:
run = _run(root, name)
- (run / "run_config.json").write_text("{}", encoding="utf-8")
- (run / "result.json").write_text(json.dumps({"run_id": name, "agents": {}}), encoding="utf-8")
+ mark_generated_output_root(run)
+ run_config: dict[str, object] = {
+ "config_file": "none",
+ "harbor": {
+ "environment": {"value": "local", "source": "test"},
+ "n_attempts": 1,
+ "stop_on_pass": False,
+ "n_concurrent": 1,
+ "timeout_multiplier": 1.0,
+ "base_image_mode": "disabled",
+ "jobs_retained": False,
+ },
+ "provider": {"name": "nvidia", "model": "test-model"},
+ "task_source": "evals_json",
+ "grading": {"mode": "default"},
+ "agents": {"opencode": {"agent": "opencode", "model": "test-model", "source": "test default"}},
+ }
+ agent_result = {
+ "model": "test-model",
+ "model_source": "test default",
+ "model_resolution": {"model": "test-model", "source": "test default"},
+ "with_skill": {"security": 0.8},
+ "without_skill": {},
+ "custom_with_skill": {},
+ "custom_without_skill": {},
+ "dimensions_with_skill": {},
+ "dimensions_without_skill": {},
+ "lift": {},
+ "custom_lift": {},
+ "pass_at_k": {"with_skill": {}, "without_skill": {}, "lift": {}},
+ "security_attribution": {},
+ "agent_runtime_failures": {"with_skill": [], "without_skill": []},
+ "trial_failures": {"with_skill": [], "without_skill": []},
+ "job_failures": {"with_skill": "", "without_skill": ""},
+ "conditions": {"with_skill": {}, "without_skill": {}},
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ "num_trials_with": 1,
+ "num_trials_without": 0,
+ "output_dir": str((run / "opencode").resolve()),
+ }
+ (run / "run_config.json").write_text(json.dumps(run_config), encoding="utf-8")
+ (run / "result.json").write_text(
+ json.dumps(
+ {
+ "skill_name": "demo",
+ "run_id": name,
+ "run_dir": str(run),
+ "result_path": str((run / "result.json").resolve()),
+ "run_config": run_config,
+ "agents": {"opencode": agent_result},
+ "attempt_policy": {
+ "max_attempts": 1,
+ "pass_threshold": 0.5,
+ "stop_on_pass": False,
+ "score_definition": "test",
+ },
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "report_status": "complete",
+ "duration_seconds": 1.0,
+ }
+ ),
+ encoding="utf-8",
+ )
return run
@@ -119,6 +185,40 @@ def _is_completed(run: Path) -> bool:
return results_location.run_directory_sort_key(run, require_completed_result=True) is not None
+def test_current_run_discovery_accepts_windows_crt_descriptor_identity(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Windows path and CRT descriptor stat identities are not comparable."""
+ run = _write_current_run(tmp_path)
+ original_read_bytes = results_location.SecureRoot.read_bytes
+
+ def read_with_windows_crt_identity(
+ secure_root: results_location.SecureRoot,
+ relative_path: Path,
+ max_bytes: int,
+ *,
+ expected: os.stat_result | None = None,
+ ) -> tuple[bytes, object]:
+ raw, opened = original_read_bytes(secure_root, relative_path, max_bytes, expected=expected)
+ return raw, SimpleNamespace(
+ st_dev=opened.st_dev + 10_000,
+ st_ino=opened.st_ino + 10_000,
+ st_mode=opened.st_mode,
+ st_nlink=opened.st_nlink,
+ st_size=opened.st_size,
+ st_uid=opened.st_uid,
+ st_mtime_ns=opened.st_mtime_ns,
+ st_ctime_ns=opened.st_ctime_ns,
+ )
+
+ monkeypatch.setattr(results_location, "_PATH_DESCRIPTOR_IDENTITIES_COMPARABLE", False, raising=False)
+ monkeypatch.setattr(output_provenance, "_PATH_DESCRIPTOR_IDENTITIES_COMPARABLE", False)
+ monkeypatch.setattr(results_location.SecureRoot, "read_bytes", read_with_windows_crt_identity)
+
+ assert _is_completed(run)
+
+
def test_rejects_current_run_with_list_valued_run_config(tmp_path: Path) -> None:
run = _write_current_run(tmp_path)
(run / "run_config.json").write_text("[]", encoding="utf-8")
diff --git a/tests/test_tier3_compare.py b/tests/test_tier3_compare.py
index d9c290ee..3298fc39 100644
--- a/tests/test_tier3_compare.py
+++ b/tests/test_tier3_compare.py
@@ -12,6 +12,7 @@ from pathlib import Path
import pytest
from skillevaluator.tier3.commands import compare_results
+from skillevaluator.tier3.output_provenance import mark_generated_output_root
def _complete_current_run(run_dir: Path) -> None:
@@ -189,6 +190,8 @@ def test_compare_same_timestamp_unique_runs_uses_result_completion_time(
(completed_later, 0.9, 200),
):
run_dir = skill_results / run_id
+ run_dir.mkdir(parents=True)
+ mark_generated_output_root(run_dir)
summary_dir = run_dir / "opencode" / "with-skill"
summary_dir.mkdir(parents=True)
(summary_dir / "summary.json").write_text(
@@ -196,8 +199,72 @@ def test_compare_same_timestamp_unique_runs_uses_result_completion_time(
encoding="utf-8",
)
result_path = run_dir / "result.json"
- (run_dir / "run_config.json").write_text("{}", encoding="utf-8")
- result_path.write_text(json.dumps({"run_id": run_id, "agents": {}}), encoding="utf-8")
+ run_config: dict[str, object] = {
+ "config_file": "none",
+ "harbor": {
+ "environment": {"value": "local", "source": "test"},
+ "n_attempts": 1,
+ "stop_on_pass": False,
+ "n_concurrent": 1,
+ "timeout_multiplier": 1.0,
+ "base_image_mode": "disabled",
+ "jobs_retained": False,
+ },
+ "provider": {"name": "nvidia", "model": "test-model"},
+ "task_source": "evals_json",
+ "grading": {"mode": "default"},
+ "agents": {"opencode": {"agent": "opencode", "model": "test-model", "source": "test default"}},
+ }
+ agent_result = {
+ "model": "test-model",
+ "model_source": "test default",
+ "model_resolution": {"model": "test-model", "source": "test default"},
+ "with_skill": {"security": score},
+ "without_skill": {},
+ "custom_with_skill": {},
+ "custom_without_skill": {},
+ "dimensions_with_skill": {},
+ "dimensions_without_skill": {},
+ "lift": {},
+ "custom_lift": {},
+ "pass_at_k": {"with_skill": {}, "without_skill": {}, "lift": {}},
+ "security_attribution": {},
+ "agent_runtime_failures": {"with_skill": [], "without_skill": []},
+ "trial_failures": {"with_skill": [], "without_skill": []},
+ "job_failures": {"with_skill": "", "without_skill": ""},
+ "conditions": {"with_skill": {}, "without_skill": {}},
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ "num_trials_with": 1,
+ "num_trials_without": 0,
+ "output_dir": str((run_dir / "opencode").resolve()),
+ }
+ (run_dir / "run_config.json").write_text(json.dumps(run_config), encoding="utf-8")
+ result_path.write_text(
+ json.dumps(
+ {
+ "skill_name": skill_path.name,
+ "run_id": run_id,
+ "run_dir": str(run_dir),
+ "result_path": str(result_path.resolve()),
+ "run_config": run_config,
+ "agents": {"opencode": agent_result},
+ "attempt_policy": {
+ "max_attempts": 1,
+ "pass_threshold": 0.5,
+ "stop_on_pass": False,
+ "score_definition": "test",
+ },
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "report_status": "complete",
+ "duration_seconds": 1.0,
+ }
+ ),
+ encoding="utf-8",
+ )
os.utime(result_path, ns=(completed_ns, completed_ns))
assert compare_results(skill_path, results_dir=tmp_path / "results") == 0
@@ -266,7 +333,7 @@ def test_compare_consumes_authenticated_timestamped_pre_status_run(
) -> None:
skill_path = tmp_path / "demo"
skill_path.mkdir()
- run_id = "20260709_010000_111_aaaaaaaaaaaa"
+ run_id = "20260709_010000"
_write_authentic_pre_status_run(tmp_path / "results" / skill_path.name, run_id)
assert compare_results(skill_path, results_dir=tmp_path / "results") == 0
diff --git a/tests/test_tier3_progress.py b/tests/test_tier3_progress.py
index f038f5c6..e6278fbf 100644
--- a/tests/test_tier3_progress.py
+++ b/tests/test_tier3_progress.py
@@ -817,6 +817,47 @@ def test_all_agent_and_baseline_arms_share_one_private_evaluator_snapshot(
assert not snapshots[0].exists()
+def test_multi_agent_run_prevalidates_baseline_alias_candidates_once(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ validation_token = object()
+ validation_calls: list[tuple[Path, tuple[Path, ...]]] = []
+ baseline_tokens: list[object | None] = []
+
+ def prevalidate(
+ skill_path: Path,
+ _reference_skills_dir: Path | None,
+ _workspace_skill_paths: list[Path] | None,
+ *,
+ excluded_roots: tuple[Path, ...],
+ ) -> object:
+ validation_calls.append((skill_path, excluded_roots))
+ return validation_token
+
+ def emit_tasks(_skill: Path, output: Path, **kwargs: Any) -> list[Path]:
+ if not kwargs["with_skill"]:
+ baseline_tokens.append(kwargs.get("_baseline_alias_validation"))
+ task = output / "case-1"
+ task.mkdir(parents=True)
+ return [task]
+
+ runner, skill = _stub_runner(monkeypatch, tmp_path, task_emitter=emit_tasks)
+ monkeypatch.setattr(runner, "_prevalidate_baseline_skill_candidates", prevalidate, raising=False)
+
+ result = runner.run_harbor_eval(
+ skill,
+ ["codex", "opencode"],
+ output_dir=tmp_path / "results",
+ agent_runtime_preflight=False,
+ )
+
+ assert "error" not in result
+ assert len(validation_calls) == 1
+ assert validation_calls[0][0] == skill
+ assert baseline_tokens == [validation_token, validation_token]
+
+
def test_runner_rejects_preexisting_run_symlink_before_child_creation(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
@@ -851,6 +892,113 @@ def test_runner_reserves_unique_run_directories_concurrently(tmp_path: Path) ->
assert all((run_dir / ".skillevaluator-generated-output").is_file() for run_dir in run_dirs)
+def test_failed_run_reservation_cleanup_refuses_a_substituted_empty_path(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ from skillevaluator.tier3.harbor import runner
+
+ results_root = tmp_path / "results"
+ moved_run = tmp_path / "moved-run"
+ monkeypatch.setattr(runner.os, "getpid", lambda: 12345)
+ monkeypatch.setattr(runner, "uuid4", lambda: SimpleNamespace(hex="abcdef012345"))
+ expected_run = results_root / "20260804_123456_12345_abcdef012345"
+
+ def substitute_then_fail(path: Path) -> None:
+ path.rename(moved_run)
+ path.mkdir()
+ raise OSError("injected marker failure")
+
+ monkeypatch.setattr(runner, "mark_generated_output_root", substitute_then_fail)
+
+ with pytest.raises(OSError, match="injected marker failure"):
+ runner._reserve_run_dir(results_root, "20260804_123456")
+
+ assert expected_run.is_dir()
+ assert moved_run.is_dir()
+
+
+def test_jobs_directory_creation_failure_returns_structured_error_and_removes_owned_run(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ runner, skill = _stub_runner(monkeypatch, tmp_path)
+ reporter = _RecordingReporter()
+ original_mkdir = Path.mkdir
+
+ def fail_jobs_mkdir(path: Path, *args: Any, **kwargs: Any) -> None:
+ if path.name == "_harbor-jobs":
+ raise OSError(errno.ENOSPC, "No space left on device")
+ original_mkdir(path, *args, **kwargs)
+
+ monkeypatch.setattr(Path, "mkdir", fail_jobs_mkdir)
+ results_root = tmp_path / "results"
+
+ result = runner.run_harbor_eval(
+ skill,
+ ["codex"],
+ output_dir=results_root,
+ progress_reporter=reporter,
+ )
+
+ assert result == {"error": ["[Errno 28] No space left on device"]}
+ assert results_root.is_dir()
+ assert list(results_root.iterdir()) == []
+ transitions = [(event.stage, event.state) for event in reporter.events]
+ assert transitions[-1] == ("run-finished", "failed")
+ assert transitions.count(("run-finished", "failed")) == 1
+
+
+def test_jobs_directory_failure_cleans_owned_empty_run_without_descriptor_backend(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ from skillevaluator.tier3 import output_provenance
+
+ runner, skill = _stub_runner(monkeypatch, tmp_path)
+ original_mkdir = Path.mkdir
+
+ def fail_jobs_mkdir(path: Path, *args: Any, **kwargs: Any) -> None:
+ if path.name == "_harbor-jobs":
+ raise OSError("injected jobs directory failure")
+ original_mkdir(path, *args, **kwargs)
+
+ monkeypatch.setattr(output_provenance, "_DESCRIPTOR_BACKEND", False)
+ monkeypatch.setattr(Path, "mkdir", fail_jobs_mkdir)
+ results_root = tmp_path / "results"
+
+ result = runner.run_harbor_eval(skill, ["codex"], output_dir=results_root)
+
+ assert result == {"error": ["injected jobs directory failure"]}
+ assert list(results_root.iterdir()) == []
+
+
+def test_jobs_directory_creation_failure_preserves_run_after_provenance_is_lost(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ runner, skill = _stub_runner(monkeypatch, tmp_path)
+ original_mkdir = Path.mkdir
+ reserved_run: list[Path] = []
+
+ def fail_after_tampering(path: Path, *args: Any, **kwargs: Any) -> None:
+ if path.name == "_harbor-jobs":
+ reserved_run.append(path.parent)
+ marker = path.parent / ".skillevaluator-generated-output"
+ marker.write_text("not an authentic marker\n", encoding="utf-8")
+ (path.parent / "preserve.txt").write_text("unowned\n", encoding="utf-8")
+ raise OSError("injected jobs directory failure")
+ original_mkdir(path, *args, **kwargs)
+
+ monkeypatch.setattr(Path, "mkdir", fail_after_tampering)
+
+ result = runner.run_harbor_eval(skill, ["codex"], output_dir=tmp_path / "results")
+
+ assert result == {"error": ["injected jobs directory failure"]}
+ assert len(reserved_run) == 1
+ assert (reserved_run[0] / "preserve.txt").read_text(encoding="utf-8") == "unowned\n"
+
+
def test_runner_publishes_latest_through_shared_atomic_helper(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
@@ -1495,24 +1643,29 @@ def test_runner_terminalizes_report_when_result_write_raises(
) -> None:
runner, skill = _stub_runner(monkeypatch, tmp_path)
reporter = _RecordingReporter()
- original_write_text = Path.write_text
+ original_replace = runner.os.replace
- def fail_result_write(path: Path, *args, **kwargs):
- if path.name == "result.json":
+ def fail_result_write(source: str | Path, destination: str | Path, *args, **kwargs):
+ if Path(destination).name == "result.json":
raise OSError("result write failed")
- return original_write_text(path, *args, **kwargs)
+ return original_replace(source, destination, *args, **kwargs)
- monkeypatch.setattr(Path, "write_text", fail_result_write)
+ monkeypatch.setattr(runner.os, "replace", fail_result_write)
+ results_root = tmp_path / "results"
with pytest.raises(OSError, match="result write failed"):
runner.run_harbor_eval(
skill,
["codex"],
- output_dir=tmp_path / "results",
+ output_dir=results_root,
keep_harbor_jobs=True,
progress_reporter=reporter,
)
+ run_dirs = [candidate for candidate in results_root.iterdir() if candidate.is_dir()]
+ assert len(run_dirs) == 1
+ assert not (run_dirs[0] / "result.json").exists()
+ assert not list(run_dirs[0].glob(".result.json.*.tmp"))
transitions = [(event.stage, event.state) for event in reporter.events]
assert ("report", "running") in transitions
assert ("report", "failed") in transitions
@@ -1520,6 +1673,31 @@ def test_runner_terminalizes_report_when_result_write_raises(
assert transitions.count(("run-finished", "failed")) == 1
+def test_runner_atomically_publishes_one_complete_final_result(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ runner, skill = _stub_runner(monkeypatch, tmp_path)
+ original_replace = runner.os.replace
+ published_payloads: list[dict[str, Any]] = []
+
+ def inspect_result_publication(source: str | Path, destination: str | Path, *args, **kwargs):
+ destination_path = Path(destination)
+ if destination_path.name == "result.json":
+ assert not destination_path.exists()
+ published_payloads.append(json.loads(Path(source).read_text(encoding="utf-8")))
+ return original_replace(source, destination, *args, **kwargs)
+
+ monkeypatch.setattr(runner.os, "replace", inspect_result_publication)
+
+ result = runner.run_harbor_eval(skill, ["codex"], output_dir=tmp_path / "results")
+
+ assert len(published_payloads) == 1
+ assert published_payloads[0]["execution_status"] == "succeeded"
+ assert published_payloads[0]["run_id"] == result["run_id"]
+ assert json.loads(Path(result["result_path"]).read_text(encoding="utf-8")) == published_payloads[0]
+
+
def test_runner_continues_when_reporter_callbacks_raise(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,