Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): scope native MCP references to verifier grants (#491)

Narendran Raghavan801fc6918ce7
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d8ef26de..991f722e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -517,6 +517,11 @@ All notable changes to SkillEvaluator are documented in this file.
to Harbor 0.25's `${VAR}` form in generated task TOML, without changing source
declarations or agent JSON configuration.
+- Validate native verifier and step-verifier MCP references against their
+ verifier environment grants, retaining agent grants for agent servers and
+ the legacy default when no separate verifier grants are supplied. Invalid
+ declarations provide field/type and `${VAR}` hints without echoing values.
+
- Keep local `/logs/artifacts`, `HARBOR_LOGS_DIR/artifacts`, and
`HARBOR_ARTIFACTS_DIR` on the same private staging tree. Artifact collection
preserves source files and retained role logs; local whole-log-tree transfers
diff --git a/src/skillevaluator/tier3/harbor/adapter.py b/src/skillevaluator/tier3/harbor/adapter.py
index 72a4c0ef..abd54d7f 100644
--- a/src/skillevaluator/tier3/harbor/adapter.py
+++ b/src/skillevaluator/tier3/harbor/adapter.py
@@ -5161,11 +5161,12 @@ def _validate_native_task_mcp_policy(
data: dict[str, Any],
*,
runtime_env: dict[str, str] | None,
+ verifier_env: dict[str, str] | None = None,
) -> None:
"""Check native MCP declarations before Harbor can resolve their headers."""
from harbor.models.task.config import MCPServerConfig
- def validate_environment(environment: object, field: str) -> None:
+ def validate_environment(environment: object, field: str, *, allowed_env: dict[str, str] | None) -> None:
# Harbor's task schema diagnoses malformed surrounding tables.
if not isinstance(environment, dict) or "mcp_servers" not in environment:
return
@@ -5173,25 +5174,36 @@ def _validate_native_task_mcp_policy(
source = f"Native Harbor task [{field}.mcp_servers]: {task_toml}"
if not isinstance(servers, list) or any(not isinstance(server, dict) for server in servers):
raise ValueError(f"{source} must be an array of MCP server tables")
+ invalid_declaration = (
+ f"{source} contains an invalid MCP server declaration; check field types and "
+ "use ${VAR} for environment placeholders"
+ )
for server in servers:
if server.keys() - MCPServerConfig.model_fields.keys():
- raise ValueError(f"{source} contains unsupported MCP server fields")
+ supported = ", ".join(sorted(MCPServerConfig.model_fields))
+ raise ValueError(
+ f"{source} contains unsupported MCP server fields; supported MCP field names: {supported}"
+ )
try:
McpServerDeclaration.model_validate(server)
except ValidationError:
- raise ValueError(f"{source} contains an invalid MCP server declaration") from None
- validate_mcp_server_declarations(servers, allowed_runtime_env=runtime_env, source_label=source)
+ raise ValueError(invalid_declaration) from None
+ validate_mcp_server_declarations(servers, allowed_runtime_env=allowed_env, source_label=source)
for server in servers:
try:
MCPServerConfig.model_validate(server)
except ValidationError:
- raise ValueError(f"{source} contains an invalid MCP server declaration") from None
+ raise ValueError(invalid_declaration) from None
- validate_environment(data.get("environment", {}), "environment")
+ validate_environment(data.get("environment", {}), "environment", allowed_env=runtime_env)
def validate_verifier(verifier: object, field: str) -> None:
if isinstance(verifier, dict):
- validate_environment(verifier.get("environment"), f"{field}.environment")
+ validate_environment(
+ verifier.get("environment"),
+ f"{field}.environment",
+ allowed_env=verifier_env if verifier_env is not None else runtime_env,
+ )
validate_verifier(data.get("verifier", {}), "verifier")
steps = data.get("steps")
@@ -5518,7 +5530,7 @@ def _native_task_workdir(
environment = data.get("environment", {}) if isinstance(data, dict) else {}
if not isinstance(environment, dict):
raise ValueError(f"Native Harbor task [environment] must be a table: {task_dir / 'task.toml'}")
- _validate_native_task_mcp_policy(task_dir / "task.toml", data, runtime_env=runtime_env)
+ _validate_native_task_mcp_policy(task_dir / "task.toml", data, runtime_env=runtime_env, verifier_env=verifier_env)
_validate_native_task_execution_compatibility(
task_dir,
data,
diff --git a/tests/test_harbor_runtime_skill_isolation.py b/tests/test_harbor_runtime_skill_isolation.py
index b3431847..b72a1094 100644
--- a/tests/test_harbor_runtime_skill_isolation.py
+++ b/tests/test_harbor_runtime_skill_isolation.py
@@ -5146,3 +5146,91 @@ def test_runtime_projection_skips_linked_generated_entries(tmp_path: Path, link_
assert (staged / "SKILL.md").is_file()
assert (staged / "scripts" / "run.sh").is_file()
assert not os.path.lexists(staged / link_name)
+
+
[email protected]("scope", ["verifier.environment", "steps.verifier.environment"])
[email protected]("grant", ["runtime_only", "verifier_only"])
+def test_native_verifier_mcp_references_use_the_verifier_environment_grants(
+ tmp_path: Path, scope: str, grant: str
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ task_toml = target / "evals" / "harbor" / "case-001" / "task.toml"
+ name = "AGENT_FLAG" if grant == "runtime_only" else "VERIFIER_FLAG"
+ step = '\n[[steps]]\nname = "step-one"\n' if scope.startswith("steps.") else ""
+ original = (
+ task_toml.read_text(encoding="utf-8")
+ + step
+ + f'\n[[{scope}.mcp_servers]]\nname = "fixture-mcp"\ntransport = "stdio"\n'
+ + f'command = "python3"\nargs = ["--flag", "${{{name}}}"]\n'
+ )
+ task_toml.write_text(original, encoding="utf-8")
+ output = tmp_path / "native-verifier-mcp-grants"
+ options = {"runtime_env": {"AGENT_FLAG": "agent-value"}, "verifier_env": {"VERIFIER_FLAG": "verifier-value"}}
+ if grant == "runtime_only":
+ with pytest.raises(ValueError, match=r"MCP|runtime|environment"):
+ stage_native_harbor_tasks(target, output, grading_mode="custom_only", **options)
+ assert not output.exists()
+ else:
+ [staged] = stage_native_harbor_tasks(target, output, grading_mode="custom_only", **options)
+ assert "${VERIFIER_FLAG}" in (staged / "task.toml").read_text(encoding="utf-8")
+ assert task_toml.read_text(encoding="utf-8") == original
+
+
+ ("scope", "verifier_env"),
+ [("environment", None), ("verifier.environment", None), ("environment", {"VERIFIER_FLAG": "verifier"})],
+)
+def test_native_mcp_keeps_agent_grants_and_legacy_verifier_defaults(
+ tmp_path: Path, scope: str, verifier_env: dict[str, str] | None
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ task_toml = target / "evals" / "harbor" / "case-001" / "task.toml"
+ task_toml.write_text(
+ task_toml.read_text(encoding="utf-8")
+ + f'\n[[{scope}.mcp_servers]]\nname = "fixture-mcp"\ntransport = "stdio"\n'
+ + 'command = "python3"\nargs = ["--flag", "${SAFE_FLAG}"]\n',
+ encoding="utf-8",
+ )
+ [staged] = stage_native_harbor_tasks(
+ target,
+ tmp_path / "legacy-native-mcp-grants",
+ grading_mode="custom_only",
+ runtime_env={"SAFE_FLAG": "safe"},
+ verifier_env=verifier_env,
+ )
+ assert "${SAFE_FLAG}" in (staged / "task.toml").read_text(encoding="utf-8")
+
+
+ ("declaration", "hint"),
+ [
+ ('headers = { Authorization = "Bearer $SAFE_MCP_TOKEN" }\n', "${VAR}"),
+ ('headers = "CANARY_INVALID_SHAPE"\n', "field types"),
+ ('private_unsupported_field = "CANARY_INVALID_SHAPE"\n', "supported MCP field names"),
+ ],
+)
+def test_native_mcp_validation_hints_do_not_echo_authored_values(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, declaration: str, hint: str
+) -> None:
+ monkeypatch.setenv("SKILLEVALUATOR_ALLOWED_MCP_HOSTS", "mcp.example.test")
+ monkeypatch.setenv("SKILLEVALUATOR_ALLOWED_MCP_SECRETS", "SAFE_MCP_TOKEN")
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ task_toml = target / "evals" / "harbor" / "case-001" / "task.toml"
+ task_toml.write_text(
+ task_toml.read_text(encoding="utf-8")
+ + '\n[[environment.mcp_servers]]\nname = "fixture-mcp"\ntransport = "streamable-http"\n'
+ + 'url = "https://mcp.example.test/mcp"\n'
+ + declaration,
+ encoding="utf-8",
+ )
+ output = tmp_path / "native-mcp-safe-hints"
+ with pytest.raises(ValueError, match=r"MCP|mcp_servers") as error:
+ stage_native_harbor_tasks(target, output, grading_mode="custom_only")
+ message = str(error.value)
+ assert hint in message
+ assert "CANARY_INVALID_SHAPE" not in message
+ assert "private_unsupported_field" not in message
+ assert not output.exists()