Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): omit local Codex credential dirs from stop-on-pass merges (#276)

Narendran Raghavancb52c19a4b4e
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 78babe59..ba8b9492 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -577,6 +577,16 @@ All notable changes to SkillEvaluator are documented in this file.
image formats, files without that structure, and every other partial reason
still leave the scan `INCOMPLETE`. SkillSpector's own High `AE1` findings for
images it could not inspect are unchanged.
+- Tier 3 `--stop-on-pass` runs with local-mode Codex no longer fail to finalize
+ when Harbor's best-effort Codex cleanup is skipped, for example after a
+ timeout or cancellation. Merging the per-attempt jobs now omits the leftover
+ `codex-home` and `codex-secrets` directories in each trial's `agent/` or
+ `steps/<step>/agent/` directory, including steps whose names span several
+ path components, such as `phase/author`, as declared in the trial's
+ `result.json`. Their linked `auth.json` made the merge fail closed, and a
+ regular-file `auth.json` was copied into the merged job with the Codex
+ credential. Symlink and hard-link checks still apply everywhere else,
+ including to the same names at any other depth.
- A `SKILL.md` over the recommended 5,000-token budget now gets a MEDIUM
`quality_efficiency` finding instead of a HIGH one. The HIGH finding failed
the quality check, and with it `validate`, `tier1`, and `quality-check`, even
diff --git a/src/skillevaluator/tier3/harbor/__init__.py b/src/skillevaluator/tier3/harbor/__init__.py
index a962bcff..d861c28f 100644
--- a/src/skillevaluator/tier3/harbor/__init__.py
+++ b/src/skillevaluator/tier3/harbor/__init__.py
@@ -35,6 +35,13 @@ CONTAINER_AGENT_IMPORT_PATHS = {
"claude-code": "skillevaluator.tier3.harbor.local_agents:SkillEvaluatorClaudeCode",
}
+# Per-trial runtime dirs local-mode Codex creates under Harbor's agent log dir.
+# ``codex-secrets/auth.json`` holds the credential (written from OPENAI_API_KEY
+# or uploaded from a host auth.json) and ``codex-home/auth.json`` links to it.
+# Both are private runtime state, never retained trial evidence.
+LOCAL_CODEX_HOME_DIRNAME = "codex-home"
+LOCAL_CODEX_SECRETS_DIRNAME = "codex-secrets"
+
# Docker wrappers retain Harbor's installers while avoiding redundant Node/npm
# packages on the branches that bootstrap their own runtime. Claude Code uses the
# container wrapper, which builds on that installer fix and adds Vertex AI
@@ -83,6 +90,8 @@ __all__ = [
"HARBOR_ENV_MODES",
"HARBOR_NATIVE_ENV_MODES",
"LOCAL_AGENT_IMPORT_PATHS",
+ "LOCAL_CODEX_HOME_DIRNAME",
+ "LOCAL_CODEX_SECRETS_DIRNAME",
"LOCAL_ENV_IMPORT_PATH",
"LOCAL_HARBOR_AGENTS",
"canonical_agent_name",
diff --git a/src/skillevaluator/tier3/harbor/local_agents.py b/src/skillevaluator/tier3/harbor/local_agents.py
index 2c8e68bb..e12da993 100644
--- a/src/skillevaluator/tier3/harbor/local_agents.py
+++ b/src/skillevaluator/tier3/harbor/local_agents.py
@@ -28,6 +28,7 @@ from harbor.models.trial.paths import EnvironmentPaths
from harbor.utils.env import parse_bool_env_value
from skillevaluator.provider_config import is_claude_5_5_or_later, is_gpt6_model
+from skillevaluator.tier3.harbor import LOCAL_CODEX_HOME_DIRNAME, LOCAL_CODEX_SECRETS_DIRNAME
from skillevaluator.tier3.harbor.installed_agents import SkillEvaluatorClaudeCode as _InstalledClaudeCode
from skillevaluator.tier3.harbor.installed_agents import SkillEvaluatorCodex
from skillevaluator.tier3.harbor.nvidia_build_bridge import (
@@ -446,8 +447,8 @@ class SkillEvaluatorGatewayCodex(SkillEvaluatorCodex):
class SkillEvaluatorLocalCodex(SkillEvaluatorGatewayCodex):
"""Use the installed local CLI with per-trial state and gateway routing."""
- _REMOTE_CODEX_HOME = PurePosixPath(EnvironmentPaths.agent_dir / "codex-home")
- _REMOTE_CODEX_SECRETS_DIR = PurePosixPath(EnvironmentPaths.agent_dir / "codex-secrets")
+ _REMOTE_CODEX_HOME = PurePosixPath(EnvironmentPaths.agent_dir / LOCAL_CODEX_HOME_DIRNAME)
+ _REMOTE_CODEX_SECRETS_DIR = PurePosixPath(EnvironmentPaths.agent_dir / LOCAL_CODEX_SECRETS_DIRNAME)
async def install(self, environment: BaseEnvironment) -> None:
result = await self.exec_as_agent(environment, command="codex --version")
diff --git a/src/skillevaluator/tier3/harbor/runner.py b/src/skillevaluator/tier3/harbor/runner.py
index 13f06395..1acf8952 100644
--- a/src/skillevaluator/tier3/harbor/runner.py
+++ b/src/skillevaluator/tier3/harbor/runner.py
@@ -78,7 +78,12 @@ from skillevaluator.tier3.evals_config import (
load_evals_config,
validate_environment_kwargs,
)
-from skillevaluator.tier3.harbor import CONTAINER_AGENT_IMPORT_PATHS, DOCKER_AGENT_IMPORT_PATHS
+from skillevaluator.tier3.harbor import (
+ CONTAINER_AGENT_IMPORT_PATHS,
+ DOCKER_AGENT_IMPORT_PATHS,
+ LOCAL_CODEX_HOME_DIRNAME,
+ LOCAL_CODEX_SECRETS_DIRNAME,
+)
from skillevaluator.tier3.harbor.adapter import (
_RUNTIME_PROCESS_CONTROL_ENV_NAMES,
_RUNTIME_PROCESS_CONTROL_ENV_PREFIXES,
@@ -134,7 +139,7 @@ from skillevaluator.tier3.harbor.report_data import (
encode_dataset_snapshot,
load_staged_harbor_dataset,
)
-from skillevaluator.tier3.harbor.secure_copy import copytree_secure
+from skillevaluator.tier3.harbor.secure_copy import IgnoreCallback, _absolute_lexical, copytree_secure
from skillevaluator.tier3.harbor.secure_docker_environment import (
SECURE_DOCKER_ENV_IMPORT_PATH,
SECURE_KATA_ENV_IMPORT_PATH,
@@ -4027,12 +4032,100 @@ def _aggregate_trial_name(
return candidate
+# Local-mode Codex runtime dirs left under a trial's agent log dir. They hold
+# the Codex credential, so attempt merges omit them instead of publishing them.
+_LOCAL_CODEX_PRIVATE_AGENT_DIRS = frozenset({LOCAL_CODEX_HOME_DIRNAME, LOCAL_CODEX_SECRETS_DIRNAME})
+
+
+def _read_json_mapping_no_follow(path: Path) -> dict[str, Any] | None:
+ """Read a regular, unlinked JSON object file, or return ``None``."""
+ try:
+ if not stat.S_ISREG(path.lstat().st_mode):
+ return None
+ fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_BINARY", 0))
+ except OSError:
+ return None
+ try:
+ with os.fdopen(fd, "rb") as handle:
+ if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode):
+ return None
+ payload = json.loads(handle.read().decode("utf-8"))
+ except (OSError, UnicodeDecodeError, json.JSONDecodeError):
+ return None
+ return payload if isinstance(payload, dict) else None
+
+
+def _trial_codex_private_dirs_ignore(job_path: Path) -> IgnoreCallback:
+ """Return a secure-copy ignore callback that drops local Codex runtime dirs.
+
+ Harbor's Codex agent links ``codex-home/auth.json`` to the credential in
+ ``codex-secrets`` and removes both only in a best-effort cleanup, so a
+ timed-out or cancelled trial can leave them behind. The exclusion is limited
+ to trial agent dirs: ``<trial>/agent``, and ``<trial>/steps/<step>/agent``
+ where Harbor's multi-step trials move it after each step. ``<step>`` is one
+ path component, or a step name the trial's ``result.json`` (or the attempt
+ root's matching ``trial_results`` entry) declares, which may span several
+ components. Every other path still passes through the secure copier's link
+ checks.
+ """
+ from skillevaluator.tier3.harbor.runtime_preflight import _agent_artifact_dirs
+
+ # copytree_secure reports directories under its own lexical source root.
+ job_root = _absolute_lexical(job_path)
+ root_trial_results: dict[str, Mapping[str, Any]] | None = None
+ declared_step_agent_dirs: dict[str, frozenset[tuple[str, ...]]] = {}
+
+ def trial_result(trial_name: str) -> Mapping[str, Any] | None:
+ nonlocal root_trial_results
+ payload = _read_json_mapping_no_follow(job_root / trial_name / "result.json")
+ if payload is not None:
+ return payload
+ # The merge falls back to the attempt root's entry for an unreadable
+ # trial result, so the declared step names come from the same place.
+ if root_trial_results is None:
+ root = _read_json_mapping_no_follow(job_root / "result.json") or {}
+ candidates = root.get("trial_results")
+ root_trial_results = {
+ candidate["trial_name"]: candidate
+ for candidate in (candidates if isinstance(candidates, list) else [])
+ if isinstance(candidate, dict) and isinstance(candidate.get("trial_name"), str)
+ }
+ return root_trial_results.get(trial_name)
+
+ def step_agent_dirs(trial_name: str) -> frozenset[tuple[str, ...]]:
+ if trial_name not in declared_step_agent_dirs:
+ payload = trial_result(trial_name) or {}
+ # Only the step results name step dirs; reuse the preflight's
+ # step-name validation so both read the same layout.
+ dirs = _agent_artifact_dirs(Path(), {"step_results": payload.get("step_results")})
+ declared_step_agent_dirs[trial_name] = frozenset(path.parts for path in dirs)
+ return declared_step_agent_dirs[trial_name]
+
+ def ignore(directory: str, names: list[str]) -> set[str]:
+ try:
+ parts = Path(directory).relative_to(job_root).parts
+ except ValueError:
+ return set()
+ if len(parts) < 2 or parts[-1] != "agent" or not _LOCAL_CODEX_PRIVATE_AGENT_DIRS.intersection(names):
+ return set()
+ trial_relative = parts[1:]
+ is_trial_agent_dir = (
+ trial_relative == ("agent",)
+ or (len(trial_relative) == 3 and trial_relative[0] == "steps")
+ or (trial_relative[0] == "steps" and trial_relative in step_agent_dirs(parts[0]))
+ )
+ return _LOCAL_CODEX_PRIVATE_AGENT_DIRS.intersection(names) if is_trial_agent_dir else set()
+
+ return ignore
+
+
def _merge_attempt_jobs(job_dirs: list[Path], aggregate_dir: Path) -> None:
"""Merge per-attempt Harbor jobs into the job directory shape collection expects.
Trial directories are copied under attempt-qualified names and the
per-attempt Harbor ``result.json`` statistics are combined so the merged
- job still satisfies :func:`validate_harbor_job_result`.
+ job still satisfies :func:`validate_harbor_job_result`. Leftover local
+ Codex credential dirs in trial agent dirs are omitted from the merge.
"""
from harbor.models.job.result import JobResult, JobStats
from harbor.models.trial.config import TrialConfig
@@ -4083,7 +4176,12 @@ def _merge_attempt_jobs(job_dirs: list[Path], aggregate_dir: Path) -> None:
raise ValueError(f"attempt Harbor job root changed before snapshot: {job_path}") from exc
if _job_path_is_link_or_reparse(job_path, before) or _job_root_fingerprint(before) != expected_fingerprint:
raise ValueError(f"attempt Harbor job root changed before snapshot: {job_path}")
- copytree_secure(job_path, snapshot, allowed_root=job_resolved)
+ copytree_secure(
+ job_path,
+ snapshot,
+ allowed_root=job_resolved,
+ ignore=_trial_codex_private_dirs_ignore(job_path),
+ )
try:
after = job_path.lstat()
except OSError as exc:
diff --git a/tests/test_harbor_runner_status.py b/tests/test_harbor_runner_status.py
index 2ef941df..7a6e094d 100644
--- a/tests/test_harbor_runner_status.py
+++ b/tests/test_harbor_runner_status.py
@@ -1481,6 +1481,200 @@ def test_merge_attempt_jobs_ignores_tmpdir_inside_attempt_job(
assert not list(tmp_path.glob(".aggregate-merge-*"))
+# Synthetic, low-entropy credential so secret scanners do not flag the fixture.
+_LOCAL_CODEX_KEY = "sk-codex-codex-codex"
+_LOCAL_CODEX_AUTH = json.dumps({"OPENAI_API_KEY": _LOCAL_CODEX_KEY})
+
+
+def _declare_trial_steps(job_dir: Path, trial_name: str, step_names: list[str], *, in_root: bool = False) -> None:
+ """Turn a fixture trial into a multi-step one that declares ``step_names``.
+
+ ``in_root`` moves the trial result into the attempt root's ``trial_results``
+ entry, which the merge reads when the trial's own result is missing.
+ """
+ trial_result_path = job_dir / trial_name / "result.json"
+ payload = json.loads(trial_result_path.read_text(encoding="utf-8"))
+ payload["agent_result"] = None
+ payload["step_results"] = [{"step_name": name} for name in step_names]
+ if not in_root:
+ trial_result_path.write_text(json.dumps(payload), encoding="utf-8")
+ return
+ trial_result_path.unlink()
+ root_result_path = job_dir / "result.json"
+ root_payload = json.loads(root_result_path.read_text(encoding="utf-8"))
+ root_payload["trial_results"] = [payload]
+ root_result_path.write_text(json.dumps(root_payload), encoding="utf-8")
+
+
+# Single-step trials keep agent logs in <trial>/agent. Harbor's multi-step
+# trials move them into <trial>/steps/<step>/agent after each step, and a step
+# name may span several path components ("phase/author").
+ ("step_root", "declared_in"),
+ [
+ (".", None),
+ ("steps/step-1", None),
+ ("steps/step-1", "trial"),
+ ("steps/phase/author", "trial"),
+ ("steps/phase/author", "root"),
+ ],
+)
[email protected]("auth_layout", ["secrets-link", "host-link", "regular-file"])
[email protected]("descriptor_backend", [False, True])
+def test_merge_attempt_jobs_omits_leftover_local_codex_credential_dirs(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ step_root: str,
+ declared_in: str | None,
+ auth_layout: str,
+ descriptor_backend: bool,
+) -> None:
+ secure_copy = importlib.import_module("skillevaluator.tier3.harbor.secure_copy")
+ if descriptor_backend and not secure_copy._DESCRIPTOR_BACKEND:
+ pytest.skip("descriptor-relative copying unavailable")
+ monkeypatch.setattr(secure_copy, "_DESCRIPTOR_BACKEND", descriptor_backend)
+ job_dir = tmp_path / "attempt-001"
+ trial_name = "case-001__trial"
+ _write_current_harbor_attempt(job_dir, trial_name=trial_name, root_completed=1)
+ if declared_in is not None:
+ step_name = step_root.removeprefix("steps/")
+ _declare_trial_steps(job_dir, trial_name, [step_name], in_root=declared_in == "root")
+ trial_dir = job_dir / trial_name
+ # Harbor's Codex agent writes (or uploads the host's) auth.json into
+ # codex-secrets and links codex-home/auth.json to it. Local mode keeps both
+ # under the trial agent dir, and a skipped best-effort cleanup leaves them.
+ agent_dir = trial_dir / step_root / "agent"
+ codex_home = agent_dir / "codex-home"
+ codex_secrets = agent_dir / "codex-secrets"
+ codex_home.mkdir(parents=True)
+ codex_secrets.mkdir()
+ secret_auth = codex_secrets / "auth.json"
+ secret_auth.write_text(_LOCAL_CODEX_AUTH, encoding="utf-8")
+ auth = codex_home / "auth.json"
+ if auth_layout == "regular-file":
+ auth.write_text(_LOCAL_CODEX_AUTH, encoding="utf-8")
+ else:
+ link_target = secret_auth
+ if auth_layout == "host-link":
+ link_target = tmp_path / "host-auth.json"
+ link_target.write_text(_LOCAL_CODEX_AUTH, encoding="utf-8")
+ try:
+ auth.symlink_to(link_target)
+ except OSError as exc: # pragma: no cover - filesystem policy
+ pytest.skip(f"file symlinks unavailable: {exc}")
+ (codex_home / "config.toml").write_text("private config\n", encoding="utf-8")
+ retained = {
+ "agent/trajectory.json": '{"messages": []}',
+ "agent/codex.txt": "codex output\n",
+ "verifier/reward.json": '{"reward": 1.0}',
+ "artifacts/output.txt": "expected output\n",
+ # The same names outside the trial agent dir are ordinary evidence.
+ "artifacts/agent/codex-home/example.txt": "authored artifact\n",
+ "artifacts/agent/codex-secrets/example.txt": "authored artifact\n",
+ }
+ for relative, contents in retained.items():
+ path = trial_dir / step_root / relative
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(contents, encoding="utf-8")
+ aggregate_dir = tmp_path / "aggregate"
+
+ runner._merge_attempt_jobs([job_dir], aggregate_dir)
+
+ merged_trial = aggregate_dir / f"{job_dir.name}__{trial_name}"
+ merged_step = merged_trial / step_root
+ assert not (merged_step / "agent" / "codex-home").exists()
+ assert not (merged_step / "agent" / "codex-secrets").exists()
+ for relative, contents in retained.items():
+ assert (merged_step / relative).read_text(encoding="utf-8") == contents
+ leaked = [
+ path.relative_to(aggregate_dir).as_posix()
+ for path in aggregate_dir.rglob("*")
+ if path.is_file() and _LOCAL_CODEX_KEY in path.read_text(encoding="utf-8", errors="ignore")
+ ]
+ assert leaked == []
+ merged_result = json.loads((merged_trial / "result.json").read_text(encoding="utf-8"))
+ assert merged_result["verifier_result"]["rewards"] == {"reward": 1.0}
+ if declared_in is not None:
+ assert [step["step_name"] for step in merged_result["step_results"]] == [step_root.removeprefix("steps/")]
+ # The source attempt is left as it was; only the aggregate omits the dirs.
+ assert auth.is_symlink() is (auth_layout != "regular-file")
+ assert auth.read_text(encoding="utf-8") == _LOCAL_CODEX_AUTH
+ assert secret_auth.read_text(encoding="utf-8") == _LOCAL_CODEX_AUTH
+ assert (codex_home / "config.toml").read_text(encoding="utf-8") == "private config\n"
+
+
+def test_merge_attempt_jobs_codex_exclusions_track_local_codex_runtime_dirs() -> None:
+ from harbor.models.trial.paths import EnvironmentPaths
+
+ from skillevaluator.tier3.harbor.local_agents import (
+ SkillEvaluatorLocalCodex,
+ SkillEvaluatorLocalNvidiaBuildCodex,
+ )
+
+ for agent_class in (SkillEvaluatorLocalCodex, SkillEvaluatorLocalNvidiaBuildCodex):
+ runtime_dirs = (agent_class._REMOTE_CODEX_HOME, agent_class._REMOTE_CODEX_SECRETS_DIR)
+ assert {path.parent for path in runtime_dirs} == {EnvironmentPaths.agent_dir}
+ assert {path.name for path in runtime_dirs} == runner._LOCAL_CODEX_PRIVATE_AGENT_DIRS
+
+
+ "relative_auth",
+ [
+ "agent/auth.json",
+ "codex-home/auth.json",
+ "codex-secrets/auth.json",
+ "agent/codex-home-other/auth.json",
+ "agent/codex-secrets-other/auth.json",
+ "artifacts/agent/codex-home/auth.json",
+ "artifacts/agent/codex-secrets/auth.json",
+ "steps/codex-home/auth.json",
+ "steps/agent/codex-home/auth.json",
+ "steps/step-1/codex-home/auth.json",
+ "steps/step-1/agent/codex-home-other/auth.json",
+ "steps/step-1/artifacts/agent/codex-home/auth.json",
+ "steps/step-1/nested/agent/codex-home/auth.json",
+ "steps/phase/author/nested/agent/codex-home/auth.json",
+ "steps/phase/author/artifacts/agent/codex-secrets/auth.json",
+ "steps/other/author/agent/codex-home/auth.json",
+ "steps/x/agent/agent/codex-home/auth.json",
+ "agent/steps/step-1/agent/codex-home/auth.json",
+ "other/step-1/agent/codex-home/auth.json",
+ ],
+)
+def test_merge_attempt_jobs_rejects_auth_links_outside_trial_codex_dirs(
+ tmp_path: Path,
+ relative_auth: str,
+) -> None:
+ job_dir = tmp_path / "attempt-001"
+ trial_dir = job_dir / "case-001__trial"
+ auth = trial_dir / relative_auth
+ auth.parent.mkdir(parents=True)
+ # Declared step names extend the exclusion only to their own agent dirs;
+ # names that would leave the steps dir are not trusted.
+ declared_steps = ["phase/author", "../agent", "x/../x/agent", "/abs"]
+ (trial_dir / "result.json").write_text(
+ json.dumps({"step_results": [{"step_name": name} for name in declared_steps]}),
+ encoding="utf-8",
+ )
+ host_auth = tmp_path / "host-auth.json"
+ host_auth.write_text("private\n", encoding="utf-8")
+ try:
+ auth.symlink_to(host_auth)
+ except OSError as exc: # pragma: no cover - filesystem policy
+ pytest.skip(f"file symlinks unavailable: {exc}")
+ aggregate_dir = tmp_path / "aggregate"
+ aggregate_dir.mkdir()
+ marker = aggregate_dir / "keep.txt"
+ marker.write_text("old aggregate", encoding="utf-8")
+
+ with pytest.raises(ValueError, match=_UNSAFE_LINK):
+ runner._merge_attempt_jobs([job_dir], aggregate_dir)
+
+ assert sorted(path.name for path in aggregate_dir.iterdir()) == ["keep.txt"]
+ assert marker.read_text(encoding="utf-8") == "old aggregate"
+ assert host_auth.read_text(encoding="utf-8") == "private\n"
+
+
def test_merge_attempt_jobs_preserves_existing_aggregate_on_unsafe_source(tmp_path: Path) -> None:
outside = tmp_path / "outside"
outside.mkdir()