Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): preserve MCP validation and local artifact paths (#472)

Narendran Raghavancf325c8dbe59
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8af24142..3b9d774a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -454,6 +454,16 @@ All notable changes to SkillEvaluator are documented in this file.
Isolated wheel tests cover SDK 1.28.1 and 2.3.0; the supported Tier 3 dependency
range remains unchanged.
+- Apply the existing MCP declaration policy to native Harbor task environments
+ before staging. Preserve approved `$VAR` header references by converting them
+ to Harbor 0.25's `${VAR}` form in generated task TOML, without changing source
+ declarations or agent JSON configuration.
+
+- Keep local `/logs/artifacts`, `HARBOR_LOGS_DIR/artifacts`, and
+ `HARBOR_ARTIFACTS_DIR` on the same private staging tree. Artifact collection
+ preserves source files and retained role logs; local whole-log-tree transfers
+ fail before modifying their destination.
+
- Repository CI can opt in to reviewed, per-skill SkillSpector baselines from
the PR or merge request's target revision. Policy files from the proposed
changes cannot approve their own exceptions.
diff --git a/docs/custom-graders.mdx b/docs/custom-graders.mdx
index 43f5421f..e399cbd4 100644
--- a/docs/custom-graders.mdx
+++ b/docs/custom-graders.mdx
@@ -158,9 +158,19 @@ a standardized JSON trajectory:
| `HARBOR_REWARD_JSON` | `/logs/verifier/reward.json` | Where to write your scores |
| `HARBOR_REWARD_TXT` | `/logs/verifier/reward.txt` | Where to write the plain-text overall score |
| `HARBOR_LOGS_DIR` | `/logs` | Root of the log tree |
+| `HARBOR_ARTIFACTS_DIR` | `/logs/artifacts` | Files to retain as trial artifacts |
| `HARBOR_TESTS_DIR` | `/tests` | Root of the staged test files |
| `HARBOR_GRADER` | `/tests/grader.py` | Where your staged grader runs from |
+In local mode, these variables point to evaluator-owned host paths.
+`HARBOR_LOGS_DIR/artifacts` and `HARBOR_ARTIFACTS_DIR` identify the same staging
+directory. Local artifact declarations must select `/logs/artifacts` or a
+specific log role such as `/logs/agent`; transferring the whole `/logs` tree
+is unsupported and fails before changing the destination.
+Role log directories retain their host locations, so use
+`HARBOR_ARTIFACTS_DIR` or `HARBOR_LOGS_DIR/artifacts` for artifacts rather than
+deriving a parent from a role directory or writing `../artifacts` from it.
+
Your grader reads Harbor's `trajectory.json` as written. Unlike the standard
grader, it does not [rebuild a missing or empty trajectory](tier3-live-evaluation.mdx#the-graded-trajectory)
from agent logs, so handle that case explicitly. A step's `message` and an
diff --git a/docs/tier3-live-evaluation.mdx b/docs/tier3-live-evaluation.mdx
index d3f225e1..acb4d178 100644
--- a/docs/tier3-live-evaluation.mdx
+++ b/docs/tier3-live-evaluation.mdx
@@ -433,6 +433,13 @@ files replaced during the read fail task generation before output mutation. On
Windows, the selected-file handle also denies concurrent writes and deletion
for the read.
+Native Harbor task `environment.mcp_servers` declarations use the same MCP
+approval policy before staging, including declarations in task-level and
+per-step verifier environments. For generated tasks,
+`evals/environment/mcp_servers.toml` accepts approved `$VAR` and `${VAR}` header
+references; generated Harbor task TOML uses the braced form required by Harbor
+0.25. Source files and the agent's MCP JSON keep their original references.
+
`evals/environment/input/` is reserved and is not copied into generated tasks.
Directory-level custom Docker `COPY input/` remains buildable for an empty entry
using an empty context directory. Missing, dynamic, linked, hardlinked, aliased,
diff --git a/src/skillevaluator/tier3/harbor/adapter.py b/src/skillevaluator/tier3/harbor/adapter.py
index 3fdc6923..72a4c0ef 100644
--- a/src/skillevaluator/tier3/harbor/adapter.py
+++ b/src/skillevaluator/tier3/harbor/adapter.py
@@ -2125,6 +2125,17 @@ def validate_mcp_server_declarations(
return servers
+def _harbor_mcp_headers(headers: Mapping[str, str]) -> dict[str, str]:
+ """Use Harbor's braced placeholders after validating the original headers."""
+ return {
+ name: _MCP_COMPLETE_PLACEHOLDER_RE.sub(
+ lambda match: f"${{{match.group(3)}}}" if match.group(3) and not match.group(1) else match.group(0),
+ value,
+ )
+ for name, value in headers.items()
+ }
+
+
def _load_mcp_servers(
skill_path: Path,
*,
@@ -2268,6 +2279,8 @@ skills_dir = "/workspace/skills"
for key, val in srv.items():
if not isinstance(key, str):
raise TypeError("MCP TOML keys must be strings")
+ if key == "headers":
+ val = _harbor_mcp_headers(val)
content += f"{_toml_quote(key)} = {_toml_value(val)}\n"
(task_dir / "mcp_servers.json").write_text(json.dumps(mcp_servers, indent=2), encoding="utf-8")
@@ -5143,6 +5156,51 @@ def _validate_native_task_host_env_boundaries(
validate_verifier(step.get("verifier", {}), f"steps[{index}].verifier")
+def _validate_native_task_mcp_policy(
+ task_toml: Path,
+ data: dict[str, Any],
+ *,
+ runtime_env: dict[str, str] | None,
+) -> None:
+ """Check native MCP declarations before Harbor can resolve their headers."""
+ from harbor.models.task.config import MCPServerConfig
+
+ def validate_environment(environment: object, field: str) -> None:
+ # Harbor's task schema diagnoses malformed surrounding tables.
+ if not isinstance(environment, dict) or "mcp_servers" not in environment:
+ return
+ servers = environment["mcp_servers"]
+ source = f"Native Harbor task [{field}.mcp_servers]: {task_toml}"
+ if not isinstance(servers, list) or any(not isinstance(server, dict) for server in servers):
+ raise ValueError(f"{source} must be an array of MCP server tables")
+ for server in servers:
+ if server.keys() - MCPServerConfig.model_fields.keys():
+ raise ValueError(f"{source} contains unsupported MCP server fields")
+ try:
+ McpServerDeclaration.model_validate(server)
+ except ValidationError:
+ raise ValueError(f"{source} contains an invalid MCP server declaration") from None
+ validate_mcp_server_declarations(servers, allowed_runtime_env=runtime_env, source_label=source)
+ for server in servers:
+ try:
+ MCPServerConfig.model_validate(server)
+ except ValidationError:
+ raise ValueError(f"{source} contains an invalid MCP server declaration") from None
+
+ validate_environment(data.get("environment", {}), "environment")
+
+ def validate_verifier(verifier: object, field: str) -> None:
+ if isinstance(verifier, dict):
+ validate_environment(verifier.get("environment"), f"{field}.environment")
+
+ validate_verifier(data.get("verifier", {}), "verifier")
+ steps = data.get("steps")
+ if isinstance(steps, list):
+ for index, step in enumerate(steps):
+ if isinstance(step, dict):
+ validate_verifier(step.get("verifier", {}), f"steps[{index}].verifier")
+
+
def _validate_native_verifier_compose_boundaries(
task_dir: Path,
data: dict[str, Any],
@@ -5460,6 +5518,7 @@ def _native_task_workdir(
environment = data.get("environment", {}) if isinstance(data, dict) else {}
if not isinstance(environment, dict):
raise ValueError(f"Native Harbor task [environment] must be a table: {task_dir / 'task.toml'}")
+ _validate_native_task_mcp_policy(task_dir / "task.toml", data, runtime_env=runtime_env)
_validate_native_task_execution_compatibility(
task_dir,
data,
diff --git a/src/skillevaluator/tier3/harbor/local_environment.py b/src/skillevaluator/tier3/harbor/local_environment.py
index ee382b4f..e2b740db 100644
--- a/src/skillevaluator/tier3/harbor/local_environment.py
+++ b/src/skillevaluator/tier3/harbor/local_environment.py
@@ -19,6 +19,7 @@ import stat
import subprocess
import sys
import tempfile
+from collections.abc import Sequence
from pathlib import Path, PurePosixPath
from typing import Any, NoReturn
@@ -546,12 +547,47 @@ class SkillEvaluatorLocalEnvironment(BaseEnvironment):
if (child := PurePosixPath(remote)) != path and child.is_relative_to(path)
)
+ @property
+ def _logs_dir(self) -> Path:
+ """Expose a coherent /logs tree without sharing collected destinations."""
+ return self._root / "runtime-logs"
+
@property
def _artifact_staging_dir(self) -> Path:
"""Keep sandbox writes separate from Harbor's collected output destinations."""
# Explicit destinations also copy the convention directory. Sharing
# their host path would make a transfer delete its own source.
- return self._root / "artifacts"
+ return self._logs_dir / "artifacts"
+
+ def _prepare_log_dirs(self) -> None:
+ for directory in (self._logs_dir, self._artifact_staging_dir):
+ directory.mkdir(parents=True, exist_ok=True)
+ if stat_is_link_or_reparse(directory.lstat()):
+ raise ValueError(f"local logs directory cannot be a symlink: {directory}")
+ # Keep role logs in Harbor's retained host locations. Only these
+ # controlled aliases let paths constructed from /logs reach them;
+ # transfers of a role still resolve directly to its real host path.
+ for role, host in (
+ ("agent", self.trial_paths.agent_dir),
+ ("verifier", self.trial_paths.verifier_dir),
+ ("user-agent", self.trial_paths.user_agent_dir),
+ ):
+ alias = self._logs_dir / role
+ target = host.resolve()
+ try:
+ metadata = alias.lstat()
+ except FileNotFoundError:
+ alias.symlink_to(target, target_is_directory=True)
+ continue
+ if not stat.S_ISLNK(metadata.st_mode) or alias.readlink() != target:
+ raise ValueError(f"local logs parent contains an unexpected role entry: {alias}")
+
+ def _reject_logs_parent_transfer(self, path: Path) -> None:
+ # Internal role aliases must never enter a downloaded artifact, and a
+ # parent upload must not replace them. Reject before changing any
+ # source/destination files; secure_copy keeps its no-follow policy.
+ if self._logs_dir.resolve().is_relative_to(path.resolve()):
+ raise ValueError("local logs parent cannot be transferred; select a role or the artifacts directory")
@classmethod
def preflight(cls) -> None:
@@ -586,7 +622,6 @@ class SkillEvaluatorLocalEnvironment(BaseEnvironment):
self.trial_paths.agent_dir,
self.trial_paths.verifier_dir,
self.trial_paths.artifacts_dir,
- self._artifact_staging_dir,
self._solution,
self._installed_agent,
self._tmp,
@@ -598,6 +633,7 @@ class SkillEvaluatorLocalEnvironment(BaseEnvironment):
self._home / ".codex",
):
path.mkdir(parents=True, exist_ok=True)
+ self._prepare_log_dirs()
self._copy_environment_bundle()
self._sandbox = local_sandbox.detect(self._sandbox_mode)
plan = self._sandbox.plan
@@ -759,6 +795,7 @@ class SkillEvaluatorLocalEnvironment(BaseEnvironment):
target = self._resolve_path(target_dir)
if not self._path_is_within_allowed_local_roots(target):
raise ValueError(f"local mode upload target is outside the run directory: {target_dir}")
+ self._reject_logs_parent_transfer(target)
# Prepare and rewrite privately so a failed upload retains the previous
# verifier. The final secure copy handles overlays and rollback.
with tempfile.TemporaryDirectory(prefix="skillevaluator-local-upload-") as directory:
@@ -817,11 +854,36 @@ class SkillEvaluatorLocalEnvironment(BaseEnvironment):
source = self._resolve_path(source_dir)
if not self._path_is_within_allowed_local_roots(source):
raise ValueError(f"local mode download source is outside the run directory: {source_dir}")
+ self._reject_logs_parent_transfer(source)
target = Path(target_dir)
if target.exists():
shutil.rmtree(target)
copytree_secure(source, target, dirs_exist_ok=True)
+ async def download_dir_with_exclusions(
+ self,
+ *,
+ source_dir: str,
+ target_dir: Path | str,
+ exclude: list[str],
+ ) -> None:
+ self._reject_logs_parent_transfer(self._resolve_path(source_dir))
+ await super().download_dir_with_exclusions(source_dir=source_dir, target_dir=target_dir, exclude=exclude)
+
+ async def download_dir_filtered(
+ self,
+ *,
+ source_dir: str,
+ target_dir: Path | str,
+ include: Sequence[str] | None = None,
+ exclude: Sequence[str] | None = None,
+ protect: Sequence[str] | None = None,
+ ) -> None:
+ self._reject_logs_parent_transfer(self._resolve_path(source_dir))
+ await super().download_dir_filtered(
+ source_dir=source_dir, target_dir=target_dir, include=include, exclude=exclude, protect=protect
+ )
+
async def exec(
self,
command: str,
@@ -1364,7 +1426,7 @@ class SkillEvaluatorLocalEnvironment(BaseEnvironment):
"TMPDIR": str(self._tmp),
"HARBOR_WORKSPACE_DIR": str(self._workspace),
"HARBOR_TESTS_DIR": str(self._tests),
- "HARBOR_LOGS_DIR": str(self.trial_paths.trial_dir),
+ "HARBOR_LOGS_DIR": str(self._logs_dir),
"HARBOR_AGENT_LOGS_DIR": str(self.trial_paths.agent_dir),
"HARBOR_VERIFIER_DIR": str(self.trial_paths.verifier_dir),
"HARBOR_ARTIFACTS_DIR": str(self._artifact_staging_dir),
@@ -1760,8 +1822,9 @@ class SkillEvaluatorLocalEnvironment(BaseEnvironment):
return [
("/logs/agent", self.trial_paths.agent_dir),
("/logs/verifier", self.trial_paths.verifier_dir),
+ ("/logs/user-agent", self.trial_paths.user_agent_dir),
("/logs/artifacts", self._artifact_staging_dir),
- ("/logs", self.trial_paths.trial_dir),
+ ("/logs", self._logs_dir),
("/workspace", self._workspace),
("/tests", self._tests),
("/solution", self._solution),
@@ -1859,11 +1922,14 @@ class SkillEvaluatorLocalEnvironment(BaseEnvironment):
if path is None:
return self._workspace
raw = str(path)
+ # Normalize logical spellings before choosing a role mapping, without
+ # following untrusted filesystem links through the virtual parent.
+ logical = str(PurePosixPath(raw))
for remote, local in self._path_map():
- if raw == remote:
+ if logical == remote:
return local
- if raw.startswith(remote + "/"):
- return local / raw[len(remote) + 1 :]
+ if logical.startswith(remote + "/"):
+ return local / logical[len(remote) + 1 :]
if raw.startswith("~/"):
return self._home / raw[2:]
candidate = Path(raw)
diff --git a/tests/test_harbor_local_environment.py b/tests/test_harbor_local_environment.py
index ccd866bc..693a5423 100644
--- a/tests/test_harbor_local_environment.py
+++ b/tests/test_harbor_local_environment.py
@@ -134,6 +134,142 @@ def test_local_exec_writes_and_reads_logical_artifact_paths(
assert environment._resolve_path("/logs/artifacts/answer.txt").read_text(encoding="utf-8") == "agent-answer"
[email protected](os.name == "nt", reason="Native Windows local mode is unsupported")
[email protected]("destination", [None, "logs/artifacts"])
+def test_local_artifact_parent_paths_publish_and_collect_step_snapshots(
+ local_output_environment: SkillEvaluatorLocalEnvironment,
+ destination: str | None,
+) -> None:
+ """Computing the artifact child from /logs must reach the publish directory."""
+ environment = local_output_environment
+ handler = ArtifactHandler(artifacts=[], logger=logging.getLogger(__name__))
+ filenames = ("parent.txt", "relative.txt", "computed.txt")
+
+ async def exercise() -> None:
+ await environment.start()
+ try:
+ for step_name, answer in (("first", "first-answer"), ("second", "second-answer")):
+ result = await environment.exec(
+ f'printf {answer} > "$HARBOR_LOGS_DIR/artifacts/parent.txt" && '
+ f"cd /logs && printf {answer} > artifacts/relative.txt && "
+ f'logs=/logs; child=artifacts; printf {answer} > "$logs/$child/computed.txt"; '
+ 'cat "$HARBOR_ARTIFACTS_DIR/parent.txt" '
+ '"$HARBOR_ARTIFACTS_DIR/relative.txt" "$HARBOR_ARTIFACTS_DIR/computed.txt"'
+ )
+ assert result.return_code == 0, result.stderr
+ assert result.stdout == answer * 3
+ handler.begin_collection()
+ entry = await handler._download_artifact(
+ source_env=environment,
+ artifacts_dir=environment.trial_paths.step_artifacts_dir(step_name),
+ artifact=ArtifactConfig(source="/logs/artifacts", destination=destination),
+ convention_source="/logs/artifacts",
+ )
+ assert entry.status == "ok"
+ for filename in filenames:
+ assert environment._resolve_path(f"/logs/artifacts/{filename}").read_text() == answer
+ finally:
+ await environment.stop(delete=True)
+
+ asyncio.run(exercise())
+
+ assert not environment._root.exists()
+ for step_name, answer in (("first", "first-answer"), ("second", "second-answer")):
+ recorded_dir = environment.trial_paths.step_artifacts_dir(step_name) / "logs" / "artifacts"
+ for filename in filenames:
+ assert (recorded_dir / filename).read_text() == answer
+
+
[email protected](os.name == "nt", reason="Native Windows local mode is unsupported")
+def test_local_parent_logs_paths_preserve_retained_role_logs(
+ local_output_environment: SkillEvaluatorLocalEnvironment,
+) -> None:
+ environment = local_output_environment
+ roles = ("agent", "verifier", "user-agent")
+ for role in roles:
+ (environment.trial_paths.trial_dir / role).mkdir(exist_ok=True)
+
+ async def exercise() -> None:
+ await environment.start()
+ try:
+ result = await environment.exec(
+ "logs=/logs; for role in agent verifier user-agent; do "
+ 'printf retained > "$logs/$role/session.jsonl" || exit; done'
+ )
+ assert result.return_code == 0, result.stderr
+ for role in roles:
+ host_logs = environment.trial_paths.trial_dir / role
+ assert environment.is_path_mounted(f"/logs/{role}", host_logs)
+ assert (host_logs / "session.jsonl").read_text() == "retained"
+ finally:
+ await environment.stop(delete=True)
+
+ asyncio.run(exercise())
+
+ for role in roles:
+ assert (environment.trial_paths.trial_dir / role / "session.jsonl").read_text() == "retained"
+
+
[email protected](os.name == "nt", reason="Native Windows local mode is unsupported")
[email protected]("transfer", ["download", "excluded", "filtered", "upload"])
+def test_local_parent_logs_transfers_fail_before_mutating_files(
+ local_output_environment: SkillEvaluatorLocalEnvironment,
+ tmp_path: Path,
+ transfer: str,
+) -> None:
+ """The virtual parent must never copy its internal aliases as user artifacts."""
+ environment = local_output_environment
+ target = tmp_path / "downloaded-logs"
+ target.mkdir()
+ marker = target / "retained.txt"
+ marker.write_text("previous download")
+ role_log = environment.trial_paths.agent_dir / "session.jsonl"
+ role_log.write_text("retained role log")
+
+ async def exercise() -> None:
+ await environment.start()
+ try:
+ with pytest.raises(ValueError, match="logs parent"):
+ if transfer == "download":
+ await environment.download_dir("/logs", target)
+ elif transfer == "excluded":
+ await environment.download_dir_with_exclusions(source_dir="/logs", target_dir=target, exclude=[])
+ elif transfer == "filtered":
+ await environment.download_dir_filtered(source_dir="/logs", target_dir=target, include=["*"])
+ else:
+ await environment.upload_dir(target, "/logs")
+ assert marker.read_text() == "previous download"
+ assert role_log.read_text() == "retained role log"
+ finally:
+ await environment.stop(delete=True)
+
+ asyncio.run(exercise())
+ assert marker.read_text() == "previous download"
+ assert role_log.read_text() == "retained role log"
+
+
[email protected](os.name == "nt", reason="Native Windows local mode is unsupported")
+def test_local_role_transfer_normalizes_logical_path_spelling(
+ local_output_environment: SkillEvaluatorLocalEnvironment,
+ tmp_path: Path,
+) -> None:
+ environment = local_output_environment
+ source = environment.trial_paths.agent_dir / "session.jsonl"
+ source.write_text("retained role log", encoding="utf-8")
+ target = tmp_path / "downloaded-role"
+
+ async def exercise() -> None:
+ await environment.start()
+ try:
+ await environment.download_dir("/logs/./agent", target)
+ finally:
+ await environment.stop(delete=True)
+
+ asyncio.run(exercise())
+ assert (target / "session.jsonl").read_text(encoding="utf-8") == "retained role log"
+ assert source.read_text(encoding="utf-8") == "retained role log"
+
+
def test_local_artifact_collection_retains_source_and_separate_step_snapshots(
local_output_environment: SkillEvaluatorLocalEnvironment,
) -> None:
diff --git a/tests/test_harbor_local_mode.py b/tests/test_harbor_local_mode.py
index 6e548034..98e92f42 100644
--- a/tests/test_harbor_local_mode.py
+++ b/tests/test_harbor_local_mode.py
@@ -115,6 +115,7 @@ def _local_environment(
{
"trial_dir": tmp_path / "trial",
"agent_dir": tmp_path / "trial" / "agent",
+ "user_agent_dir": tmp_path / "trial" / "user-agent",
"verifier_dir": tmp_path / "trial" / "verifier",
"artifacts_dir": tmp_path / "trial" / "artifacts",
"reward_json_path": tmp_path / "trial" / "verifier" / "reward.json",
diff --git a/tests/test_harbor_runtime_skill_isolation.py b/tests/test_harbor_runtime_skill_isolation.py
index 13f01f15..b3431847 100644
--- a/tests/test_harbor_runtime_skill_isolation.py
+++ b/tests/test_harbor_runtime_skill_isolation.py
@@ -576,6 +576,165 @@ def test_native_tasks_reject_unstaged_parent_env_templates_in_every_harbor_scope
)
+ "scope",
+ [
+ "environment",
+ "verifier.environment",
+ "steps.verifier.environment",
+ ],
+)
+ "header",
+ ["Bearer ${OPENAI_API_KEY}", "Bearer ${DEDICATED_MCP_TOKEN}", "plainPrivateCredential2024"],
+)
+def test_native_task_mcp_policy_is_checked_before_publication(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, scope: str, header: str
+) -> None:
+ monkeypatch.delenv("SKILLEVALUATOR_ALLOWED_MCP_HOSTS", raising=False)
+ monkeypatch.delenv("SKILLEVALUATOR_ALLOWED_MCP_SECRETS", raising=False)
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ task_toml = target / "evals" / "harbor" / "case-001" / "task.toml"
+ step = '\n[[steps]]\nname = "step-one"\n' if scope.startswith("steps.") else ""
+ original = (
+ task_toml.read_text(encoding="utf-8")
+ + step
+ + f'\n[[{scope}.mcp_servers]]\nname = "remote-mcp"\ntransport = "streamable-http"\n'
+ + 'url = "https://mcp.example.test/mcp"\n'
+ + f"headers = {{ Authorization = {json.dumps(header)} }}\n"
+ )
+ task_toml.write_text(original, encoding="utf-8")
+ output = tmp_path / "native-mcp-output"
+ output.mkdir()
+ (output / "previous.txt").write_text("previous publication", encoding="utf-8")
+
+ with pytest.raises(ValueError, match=r"MCP|mcp_servers") as error:
+ stage_native_harbor_tasks(target, output, grading_mode="custom_only")
+
+ assert "plainPrivateCredential2024" not in str(error.value)
+ assert task_toml.read_text(encoding="utf-8") == original
+ assert [path.name for path in output.iterdir()] == ["previous.txt"]
+ assert (output / "previous.txt").read_text(encoding="utf-8") == "previous publication"
+
+
[email protected]("header", ["Bearer $DEDICATED_MCP_TOKEN", "Bearer ${DEDICATED_MCP_TOKEN}"])
+def test_generated_mcp_headers_parse_in_pinned_harbor_without_changing_rich_json(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, header: str
+) -> None:
+ monkeypatch.setenv("SKILLEVALUATOR_ALLOWED_MCP_HOSTS", "mcp.example.test")
+ monkeypatch.setenv("SKILLEVALUATOR_ALLOWED_MCP_SECRETS", "DEDICATED_MCP_TOKEN")
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ source = target / "evals" / "environment" / "mcp_servers.toml"
+ original = (
+ '[[mcp_servers]]\nname = "remote-mcp"\ntransport = "streamable-http"\n'
+ 'url = "https://mcp.example.test/mcp"\n'
+ f"headers = {{ Authorization = {json.dumps(header)} }}\n"
+ )
+ source.write_text(original, encoding="utf-8")
+
+ [staged] = generate_harbor_tasks(target, tmp_path / "generated-mcp")
+
+ config = TaskConfig.model_validate_toml((staged / "task.toml").read_text(encoding="utf-8"))
+ assert config.environment.mcp_servers[0].headers == {"Authorization": "Bearer ${DEDICATED_MCP_TOKEN}"}
+ rich_servers = json.loads((staged / "mcp_servers.json").read_text(encoding="utf-8"))
+ assert rich_servers[0]["headers"] == {"Authorization": header}
+ assert source.read_text(encoding="utf-8") == original
+
+
+ "authored_config",
+ [
+ 'mcp_servers = { name = "remote-mcp" }\n',
+ 'mcp_servers = ["private-malformed-value"]\n',
+ '[[environment.mcp_servers]]\nname = "remote-mcp"\nurl = "https://mcp.example.test/mcp"\n'
+ 'headers = "private-malformed-value"\n',
+ '[[environment.mcp_servers]]\nname = "remote-mcp"\nurl = "https://mcp.example.test/mcp"\n'
+ 'unsupported_field = "private-malformed-value"\n',
+ ],
+)
+def test_native_tasks_reject_malformed_or_unsupported_mcp_without_echoing_values(
+ tmp_path: Path, authored_config: str
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ task_toml = target / "evals" / "harbor" / "case-001" / "task.toml"
+ original = task_toml.read_text(encoding="utf-8") + authored_config
+ task_toml.write_text(original, encoding="utf-8")
+ output = tmp_path / "native-invalid-mcp"
+
+ with pytest.raises(ValueError, match="mcp_servers") as error:
+ stage_native_harbor_tasks(target, output, grading_mode="custom_only")
+
+ assert "private-malformed-value" not in str(error.value)
+ assert task_toml.read_text(encoding="utf-8") == original
+ assert not output.exists()
+
+
[email protected]("transport", ["streamable-http", "stdio"])
+def test_native_tasks_preserve_approved_mcp_and_safe_runtime_references(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, transport: str
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ task_toml = target / "evals" / "harbor" / "case-001" / "task.toml"
+ if transport == "streamable-http":
+ monkeypatch.setenv("SKILLEVALUATOR_ALLOWED_MCP_HOSTS", "mcp.example.test")
+ monkeypatch.setenv("SKILLEVALUATOR_ALLOWED_MCP_SECRETS", "DEDICATED_MCP_TOKEN")
+ declaration = (
+ 'url = "https://mcp.example.test/mcp"\nheaders = { Authorization = "Bearer ${DEDICATED_MCP_TOKEN}" }\n'
+ )
+ else:
+ declaration = 'command = "python3"\nargs = ["--flag", "${SAFE_FLAG}"]\n'
+ original = (
+ task_toml.read_text(encoding="utf-8")
+ + f'[[environment.mcp_servers]]\nname = "fixture-mcp"\ntransport = "{transport}"\n'
+ + declaration
+ )
+ task_toml.write_text(original, encoding="utf-8")
+
+ [staged] = stage_native_harbor_tasks(target, tmp_path / "native-approved-mcp", runtime_env={"SAFE_FLAG": "yes"})
+
+ config = TaskConfig.model_validate_toml((staged / "task.toml").read_text(encoding="utf-8"))
+ server = config.environment.mcp_servers[0]
+ if transport == "streamable-http":
+ assert server.headers == {"Authorization": "Bearer ${DEDICATED_MCP_TOKEN}"}
+ else:
+ assert server.args == ["--flag", "${SAFE_FLAG}"]
+ assert task_toml.read_text(encoding="utf-8") == original
+
+
+ "header",
+ [
+ "Bearer $${DEDICATED_MCP_TOKEN}",
+ "Bearer $$DEDICATED_MCP_TOKEN",
+ "Bearer ${DEDICATED_MCP_TOKEN:-private-fallback}",
+ "Bearer ${DEDICATED_MCP_TOKEN",
+ ],
+)
+def test_generated_mcp_header_normalization_keeps_escaped_and_malformed_templates_rejected(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, header: str
+) -> None:
+ monkeypatch.setenv("SKILLEVALUATOR_ALLOWED_MCP_HOSTS", "mcp.example.test")
+ monkeypatch.setenv("SKILLEVALUATOR_ALLOWED_MCP_SECRETS", "DEDICATED_MCP_TOKEN")
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ source = target / "evals" / "environment" / "mcp_servers.toml"
+ original = (
+ '[[mcp_servers]]\nname = "remote-mcp"\ntransport = "streamable-http"\n'
+ 'url = "https://mcp.example.test/mcp"\n'
+ f"headers = {{ Authorization = {json.dumps(header)} }}\n"
+ )
+ source.write_text(original, encoding="utf-8")
+ output = tmp_path / "generated-invalid-mcp"
+
+ with pytest.raises(ValueError, match="MCP"):
+ generate_harbor_tasks(target, output)
+
+ assert source.read_text(encoding="utf-8") == original
+ assert not output.exists()
+
+
def test_native_tasks_allow_only_role_staged_host_env_templates(tmp_path: Path) -> None:
_, target, _, _ = _write_projection_fixture(tmp_path)
_write_minimal_native_task(target)