Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Security regressions for the environment passed to the Harbor subprocess."""
from __future__ import annotations
import json
import os
import shutil
import subprocess
import threading
import tomllib
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import pytest
from harbor.utils.env import resolve_env_vars
from skillevaluator.provider_config import ProviderConfig, ProviderConfigurationError, resolve_llm_provider
from skillevaluator.tier3.harbor import runner, runtime_preflight
from skillevaluator.tier3.harbor.adapter import _verifier_env_vars
from skillevaluator.tier3.harbor.metrics import DEFAULT_METRICS, LEGACY_SCORE_POLICY
from skillevaluator.tier3_environments import HARBOR_NATIVE_ENV_MODES
def _provider(provider: str = "openai") -> ProviderConfig:
return ProviderConfig(
provider=provider,
model="test-model",
api_key="provider-key",
base_url="https://provider.example/v1",
litellm_model="openai/test-model",
region="us-west-2" if provider == "bedrock" else None,
)
def test_docker_opencode_nv_build_uses_operator_key_without_skill_runtime_env(
monkeypatch: pytest.MonkeyPatch,
) -> None:
provider = _provider("nv_build")
monkeypatch.setattr(runner.os, "environ", {"PATH": "/usr/bin", "NVIDIA_API_KEY": "nvapi-test"})
plans = runner._resolve_agent_runtime_plan(
provider=provider,
agents=["opencode"],
models={"opencode": "nvidia/meta/llama-3.1-8b-instruct"},
configured_runtime_env={},
env_mode="docker",
)
plan = plans["opencode"]
assert plan.staged_env == {"NVIDIA_API_KEY": "${NVIDIA_API_KEY}"}
assert plan.subprocess_env["NVIDIA_API_KEY"] == "provider-key"
assert "OPENAI_API_KEY" not in plan.subprocess_env
def test_harbor_environment_keeps_windows_program_dirs_and_python_utf8_mode(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# Docker Desktop finds its CLI plugins under %ProgramFiles%, and Harbor needs the
# UTF-8 mode variables on non-UTF-8 Windows locales.
windows_env = {
"PROGRAMDATA": r"C:\ProgramData",
"PROGRAMFILES": r"C:\Program Files",
"PROGRAMW6432": r"C:\Program Files",
"PYTHONIOENCODING": "utf-8",
"PYTHONUTF8": "1",
}
monkeypatch.setattr(runner.os, "environ", {"PATH": r"C:\Windows", "UNRELATED_TOKEN": "secret", **windows_env})
env = runner._harbor_subprocess_environment(
env_mode="docker",
provider=_provider("nv_build"),
configured_runtime_env={},
provider_env={},
)
assert {name: env.get(name) for name in windows_env} == windows_env
assert "UNRELATED_TOKEN" not in env
@pytest.mark.parametrize(
("provider_name", "runtime_model", "catalog_model", "litellm_prefix"),
[
("nv_build", "nvidia/meta/llama-3.1-8b-instruct", "meta/llama-3.1-8b-instruct", "openai"),
("openai", "openai/gpt-4.1-mini", "gpt-4.1-mini", "openai"),
("openai-compatible", "openai/custom-model", "custom-model", "openai"),
("anthropic", "anthropic/claude-sonnet-test", "claude-sonnet-test", "anthropic"),
],
)
def test_opencode_runtime_plan_separates_runtime_and_catalog_model_names(
provider_name: str,
runtime_model: str,
catalog_model: str,
litellm_prefix: str,
) -> None:
provider = ProviderConfig(
provider=provider_name,
model=catalog_model,
api_key="provider-key",
base_url="https://provider.example/v1",
litellm_model=f"{litellm_prefix}/{catalog_model}",
)
plan = runner._resolve_agent_runtime_plan(
provider=provider,
agents=["opencode"],
models={"opencode": runtime_model},
configured_runtime_env={},
env_mode="docker",
model_sources={"opencode": "CLI"},
)["opencode"]
assert plan.model == runtime_model
assert plan.provider.model == catalog_model
assert plan.provider.litellm_model == f"{litellm_prefix}/{catalog_model}"
@pytest.mark.parametrize(
("provider_name", "agent", "model", "litellm_prefix"),
[
("openai-compatible", "codex", "openai/gpt-oss", "openai"),
("anthropic", "claude-code", "anthropic/claude-custom", "anthropic"),
],
)
def test_non_opencode_runtime_plan_preserves_slash_prefixed_raw_model_ids(
provider_name: str,
agent: str,
model: str,
litellm_prefix: str,
) -> None:
provider = ProviderConfig(
provider=provider_name,
model=model,
api_key="provider-key",
base_url="https://provider.example/v1",
litellm_model=f"{litellm_prefix}/{model}",
)
plan = runner._resolve_agent_runtime_plan(
provider=provider,
agents=[agent],
models={agent: model},
configured_runtime_env={},
env_mode="docker",
)[agent]
assert plan.model == model
assert plan.provider.model == model
assert plan.provider.litellm_model == f"{litellm_prefix}/{model}"
@pytest.mark.parametrize("agent", ["codex", "claude-code"])
def test_nvidia_build_docker_bridge_plan_keeps_provider_key_out_of_task_env(agent: str) -> None:
plan = runner._resolve_agent_runtime_plan(
provider=_provider("nv_build"),
agents=[agent],
models={agent: "openai/gpt-oss-120b"},
configured_runtime_env={},
env_mode="docker",
)[agent]
assert plan.staged_env == {}
assert plan.subprocess_env["NVIDIA_API_KEY"] == "provider-key"
assert plan.provider.provider == "nv_build"
assert plan.provider.model == "openai/gpt-oss-120b"
@pytest.mark.parametrize(
"owned_name",
[
"NVIDIA_API_KEY",
"OPENAI_API_KEY",
"OPENAI_BASE_URL",
"ANTHROPIC_API_KEY",
"ANTHROPIC_BASE_URL",
],
)
def test_skill_config_cannot_override_operator_owned_agent_credentials(owned_name: str) -> None:
with pytest.raises(ValueError, match=rf"operator-owned.*{owned_name}|{owned_name}.*operator-owned"):
runner._resolve_agent_runtime_plan(
provider=_provider("nv_build"),
agents=["opencode"],
models={"opencode": "nvidia/model"},
configured_runtime_env={owned_name: "attacker-value"},
env_mode="docker",
)
@pytest.mark.parametrize(
"source_name",
[
"NVIDIA_API_KEY",
"OPENAI_API_KEY",
"OPENAI_BASE_URL",
"ANTHROPIC_API_KEY",
"ANTHROPIC_BASE_URL",
"LLM_JUDGE_MODEL",
"SKILL_EVAL_JUDGE_MODEL",
"AWS_SECRET_ACCESS_KEY",
"E2B_API_KEY",
"DOCKER_HOST",
],
)
def test_skill_config_cannot_alias_operator_owned_credentials(
monkeypatch: pytest.MonkeyPatch,
source_name: str,
) -> None:
monkeypatch.setenv(source_name, "operator-secret")
resolved, errors = runner._resolve_runtime_env(
{"INNOCENT_NAME": f"${{{source_name}}}"},
env_mode="docker",
)
assert resolved == {}
assert errors and "operator-owned" in errors[0]
assert source_name in errors[0]
@pytest.mark.parametrize(
"runtime_env",
[
{"SAFE_ALIAS": "${PROJECT_DEPLOY_TOKEN}"},
{"PROJECT_DEPLOY_TOKEN": "${PROJECT_DEPLOY_TOKEN}"},
{"safe_alias": "${ProjectDeployToken}"},
],
)
def test_skill_config_cannot_stage_conventional_unlisted_secret_names(
monkeypatch: pytest.MonkeyPatch,
runtime_env: dict[str, str],
) -> None:
for value in runtime_env.values():
source_name = value.removeprefix("${").removesuffix("}")
monkeypatch.setenv(source_name, "synthetic-secret")
resolved, errors = runner._resolve_runtime_env(runtime_env, env_mode="docker")
assert resolved == {}
assert errors
assert "operator-owned" in errors[0] or "host process" in errors[0]
assert "synthetic-secret" not in " ".join(errors)
@pytest.mark.parametrize("source_name", ["LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL"])
def test_skill_config_cannot_alias_judge_model_with_default_expansion(
monkeypatch: pytest.MonkeyPatch,
source_name: str,
) -> None:
monkeypatch.setenv(source_name, "operator-model")
resolved, errors = runner._resolve_runtime_env(
{"INNOCENT_NAME": f"${{{source_name}:-skill-fallback}}"},
env_mode="docker",
)
assert resolved == {}
assert errors and "operator-owned" in errors[0]
assert source_name in errors[0]
@pytest.mark.parametrize(
"name",
[
"LLM_JUDGE_MODEL",
"SKILL_EVAL_JUDGE_MODEL",
"SKILL_EVAL_LLM_BASE_URL",
"SKILL_EVAL_LLM_PROVIDER",
"SKILL_EVAL_LLM_API_KEY",
],
)
def test_skill_config_cannot_control_evaluator_or_judge_routing(name: str) -> None:
resolved, errors = runner._resolve_runtime_env(
{name: "https://attacker.example/v1"},
env_mode="docker",
)
assert resolved == {}
assert errors and "host process" in errors[0]
@pytest.mark.parametrize(
"name",
[
"LLM_JUDGE_MODEL",
"SKILL_EVAL_JUDGE_MODEL",
"LLM_JUDGE_FALLBACK_MODELS",
"SKILL_EVAL_JUDGE_CHARACTER_FACTS",
"LLM_JUDGE_FALLBACK_POLICY",
"LLM_JUDGE_RETRY_POLICY",
"SKILL_EVAL_LLM_REFUSAL_FALLBACK_MODEL",
],
)
@pytest.mark.parametrize("provider_name", ["openai", "openai-compatible", "anthropic", "bedrock", "nv_build"])
@pytest.mark.parametrize(
("configured", "expected"),
[
("judge-model", "judge-model"),
(" judge-model ", "judge-model"),
("", None),
(" ", None),
],
)
def test_provider_environment_forwards_each_host_judge_model_override(
monkeypatch: pytest.MonkeyPatch,
name: str,
provider_name: str,
configured: str,
expected: str | None,
) -> None:
monkeypatch.setattr(runner.os, "environ", {name: configured})
environment = runner._provider_environment(_provider(provider_name))
assert environment.get(name) == expected
@pytest.mark.parametrize(
"name",
[
"SKILL_EVAL_LLM_MAX_RETRIES",
"SKILL_EVAL_LLM_RETRY_BASE_DELAY",
"SKILL_EVAL_LLM_RETRY_MAX_DELAY",
],
)
@pytest.mark.parametrize("provider_name", ["openai", "openai-compatible", "anthropic", "bedrock", "nv_build"])
@pytest.mark.parametrize(
("configured", "expected"),
[
("0", "0"),
(" 0.25 ", "0.25"),
("", None),
(" ", None),
],
)
def test_provider_environment_forwards_each_host_retry_override(
monkeypatch: pytest.MonkeyPatch,
name: str,
provider_name: str,
configured: str,
expected: str | None,
) -> None:
"""Verify _provider_environment forwards stripped host retry settings and omits blank values."""
monkeypatch.setattr(runner.os, "environ", {name: configured})
environment = runner._provider_environment(_provider(provider_name))
assert environment.get(name) == expected
def test_resolve_agent_runtime_plan_retains_retry_settings_in_subprocess_env(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Verify _resolve_agent_runtime_plan forwards host retry settings to Harbor subprocess_env."""
monkeypatch.setattr(
runner.os,
"environ",
{
"PATH": "/usr/bin",
"SKILL_EVAL_LLM_MAX_RETRIES": "0",
"SKILL_EVAL_LLM_RETRY_BASE_DELAY": "0.1",
"SKILL_EVAL_LLM_RETRY_MAX_DELAY": "0.5",
},
)
plans = runner._resolve_agent_runtime_plan(
provider=_provider("openai"),
agents=["codex"],
models={"codex": "gpt-4o-mini"},
configured_runtime_env={},
env_mode="docker",
)
subprocess_env = plans["codex"].subprocess_env
assert subprocess_env["SKILL_EVAL_LLM_MAX_RETRIES"] == "0"
assert subprocess_env["SKILL_EVAL_LLM_RETRY_BASE_DELAY"] == "0.1"
assert subprocess_env["SKILL_EVAL_LLM_RETRY_MAX_DELAY"] == "0.5"
assert "SKILL_EVAL_LLM_MAX_RETRIES" not in plans["codex"].staged_env
@pytest.mark.parametrize("source_name", ["LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL"])
def test_runtime_env_rejects_operator_owned_windows_style_variable_references(
monkeypatch: pytest.MonkeyPatch,
source_name: str,
) -> None:
monkeypatch.setenv(source_name, "host-value")
resolved, errors = runner._resolve_runtime_env(
{"SAFE_ALIAS": f"%{source_name}%"},
env_mode="docker",
)
assert resolved == {}
assert errors and source_name in errors[0]
assert "operator-owned" in errors[0]
def test_runtime_env_preserves_platform_expansion_for_non_owned_percent_references(
monkeypatch: pytest.MonkeyPatch,
) -> None:
expanded: list[str] = []
def expandvars(value: str) -> str:
expanded.append(value)
return value.replace("%HOST_AGENT_LABEL%", "host-value")
monkeypatch.setattr(runner.os.path, "expandvars", expandvars)
resolved, errors = runner._resolve_runtime_env(
{"SAFE_ALIAS": "%HOST_AGENT_LABEL%"},
env_mode="docker",
)
assert errors == []
assert resolved == {"SAFE_ALIAS": "host-value"}
assert expanded == ["%HOST_AGENT_LABEL%"]
def test_runtime_env_allows_mode_unrelated_backend_name() -> None:
resolved, errors = runner._resolve_runtime_env(
{"API_HOST": "ordinary-task-value"},
env_mode="docker",
)
assert errors == []
assert resolved == {"API_HOST": "ordinary-task-value"}
def test_runtime_env_rejects_selected_backend_control_name() -> None:
resolved, errors = runner._resolve_runtime_env(
{"API_HOST": "attacker-controlled"},
env_mode="beam",
)
assert resolved == {}
assert errors == ["harbor.runtime_env.API_HOST controls the host process and is not allowed"]
def test_custom_only_does_not_force_the_standard_judge_model_into_custom_verifiers() -> None:
provider_env = {
"LLM_JUDGE_MODEL": "legacy-judge-model",
"SKILL_EVAL_JUDGE_MODEL": "judge-model",
}
assert runner._job_judge_verifier_env(provider_env, "custom_only") == {}
assert runner._job_judge_subprocess_env(provider_env, "custom_only") == {}
assert runner._judge_model_config(_provider("nv_build"), provider_env, "custom_only") == {
"enabled": False,
}
@pytest.mark.parametrize(
("provider_env", "expected"),
[
(
{"SKILL_EVAL_JUDGE_MODEL": "operator-model"},
{
"LLM_JUDGE_MODEL": "${SKILL_EVAL_JUDGE_MODEL}",
"SKILL_EVAL_JUDGE_MODEL": "${SKILL_EVAL_JUDGE_MODEL}",
},
),
(
{"LLM_JUDGE_MODEL": "legacy-model", "SKILL_EVAL_JUDGE_MODEL": "operator-model"},
{
"LLM_JUDGE_MODEL": "${LLM_JUDGE_MODEL}",
"SKILL_EVAL_JUDGE_MODEL": "${LLM_JUDGE_MODEL}",
},
),
],
)
def test_standard_judge_override_occupies_the_highest_precedence_verifier_key(
provider_env: dict[str, str],
expected: dict[str, str],
) -> None:
assert runner._job_judge_verifier_env(provider_env, "default") == expected
@pytest.mark.parametrize("grading_mode", ["default", "default_plus_custom"])
def test_standard_judge_fallback_is_forwarded_only_through_the_verifier_job(
grading_mode: str,
) -> None:
provider_env = {"LLM_JUDGE_FALLBACK_MODELS": "fallback-one,fallback-two"}
assert runner._job_judge_verifier_env(provider_env, grading_mode) == {
"LLM_JUDGE_FALLBACK_MODELS": "${LLM_JUDGE_FALLBACK_MODELS}",
}
assert runner._job_judge_subprocess_env(provider_env, grading_mode) == {
"LLM_JUDGE_FALLBACK_MODELS": "fallback-one,fallback-two",
}
assert runner._job_judge_verifier_env(provider_env, "custom_only") == {}
assert runner._job_judge_subprocess_env(provider_env, "custom_only") == {}
@pytest.mark.parametrize(
"host_env",
[
{"LLM_JUDGE_MODEL": "host-legacy"},
{"SKILL_EVAL_JUDGE_MODEL": "host-canonical"},
{"LLM_JUDGE_MODEL": "host-legacy", "SKILL_EVAL_JUDGE_MODEL": "host-canonical"},
],
)
def test_standard_judge_override_replaces_both_native_aliases_before_harbor_resolution(
monkeypatch: pytest.MonkeyPatch,
host_env: dict[str, str],
) -> None:
for name in ("LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL"):
monkeypatch.delenv(name, raising=False)
for name, value in host_env.items():
monkeypatch.setenv(name, value)
provider_env = runner._provider_environment(_provider())
task_env = {
"LLM_JUDGE_MODEL": "task-legacy",
"SKILL_EVAL_JUDGE_MODEL": "${SKILL_EVAL_JUDGE_MODEL}",
}
merged = {**task_env, **runner._job_judge_verifier_env(provider_env, "default")}
resolved = resolve_env_vars(merged)
selected = host_env.get("LLM_JUDGE_MODEL") or host_env["SKILL_EVAL_JUDGE_MODEL"]
assert resolved == {
"LLM_JUDGE_MODEL": selected,
"SKILL_EVAL_JUDGE_MODEL": selected,
}
@pytest.mark.parametrize("grading_mode", ["default", "default_plus_custom"])
@pytest.mark.parametrize(
"host_env",
[
{"LLM_JUDGE_MODEL": "host-legacy"},
{"SKILL_EVAL_JUDGE_MODEL": "host-canonical"},
{"LLM_JUDGE_MODEL": "host-legacy", "SKILL_EVAL_JUDGE_MODEL": "host-canonical"},
],
)
def test_standard_judge_override_resolves_both_aliases_for_separate_verifier_startup(
monkeypatch: pytest.MonkeyPatch,
grading_mode: str,
host_env: dict[str, str],
) -> None:
for name in ("LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL"):
monkeypatch.delenv(name, raising=False)
for name, value in host_env.items():
monkeypatch.setenv(name, value)
provider_env = runner._provider_environment(_provider())
subprocess_aliases = runner._job_judge_subprocess_env(provider_env, grading_mode)
with monkeypatch.context() as resolver_env:
for name in ("LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL"):
resolver_env.delenv(name, raising=False)
for name, value in subprocess_aliases.items():
resolver_env.setenv(name, value)
resolved = resolve_env_vars(
{
"LLM_JUDGE_MODEL": "${LLM_JUDGE_MODEL}",
"SKILL_EVAL_JUDGE_MODEL": "${SKILL_EVAL_JUDGE_MODEL}",
}
)
selected = host_env.get("LLM_JUDGE_MODEL") or host_env["SKILL_EVAL_JUDGE_MODEL"]
assert resolved == {
"LLM_JUDGE_MODEL": selected,
"SKILL_EVAL_JUDGE_MODEL": selected,
}
@pytest.mark.parametrize(
("configured_model", "expected_source"),
[(None, "provider default"), ("operator-model", "SKILL_EVAL_LLM_MODEL")],
)
def test_judge_model_config_distinguishes_provider_default_from_explicit_model(
monkeypatch: pytest.MonkeyPatch,
configured_model: str | None,
expected_source: str,
) -> None:
if configured_model is None:
monkeypatch.delenv("SKILL_EVAL_LLM_MODEL", raising=False)
else:
monkeypatch.setenv("SKILL_EVAL_LLM_MODEL", configured_model)
provider = _provider("openai")
config = runner._judge_model_config(provider, runner._provider_environment(provider), "default")
assert config == {
"enabled": True,
"provider": "openai",
"model": "test-model",
"source": expected_source,
"override_applied": False,
"reasoning_effort": None,
}
@pytest.mark.parametrize(
("provider_name", "judge_model", "effort"),
[
("openai", "gpt-6.1-sol", "medium"),
("anthropic", "claude-opus-5-5", "medium"),
("openai", "gpt-5.6-sol", None),
("bedrock", "us.anthropic.claude-opus-5-5", None),
],
)
def test_judge_model_config_records_the_judge_reasoning_effort(
provider_name: str, judge_model: str, effort: str | None
) -> None:
provider = _provider(provider_name)
config = runner._judge_model_config(provider, {"LLM_JUDGE_MODEL": judge_model}, "default")
assert config["reasoning_effort"] == effort
def test_mixed_agents_receive_disjoint_credentials(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
runner.os,
"environ",
{
"PATH": "/usr/bin",
"NVIDIA_API_KEY": "nvapi-test",
"ANTHROPIC_API_KEY": "anthropic-test",
"OPENAI_API_KEY": "openai-test",
"OPENAI_BASE_URL": "https://api.openai.com/v1",
},
)
plans = runner._resolve_agent_runtime_plan(
provider=_provider("nv_build"),
agents=["opencode", "claude-code", "codex"],
models={
"opencode": "nvidia/nvidia/nemotron-3-nano-30b-a3b",
"claude-code": "nvidia/nemotron-3.5-lightning-30b-a3b",
"codex": "nvidia/nemotron-3-nano-30b-a3b",
},
configured_runtime_env={},
env_mode="docker",
model_sources={"opencode": "provider", "claude-code": "CLI", "codex": "CLI"},
)
assert plans["opencode"].staged_env == {"NVIDIA_API_KEY": "${NVIDIA_API_KEY}"}
assert plans["claude-code"].staged_env == {}
assert plans["codex"].staged_env == {}
assert plans["opencode"].provider.provider == "nv_build"
assert plans["claude-code"].provider.provider == "nv_build"
assert plans["codex"].provider.provider == "nv_build"
assert "ANTHROPIC_API_KEY" not in plans["opencode"].subprocess_env
assert "OPENAI_API_KEY" not in plans["opencode"].subprocess_env
assert "NVIDIA_API_KEY" in plans["claude-code"].subprocess_env # evaluator verifier only
assert "OPENAI_API_KEY" not in plans["claude-code"].subprocess_env
assert "ANTHROPIC_API_KEY" not in plans["codex"].subprocess_env
with pytest.raises(TypeError):
plans["opencode"].staged_env["MUTATE"] = "forbidden" # type: ignore[index]
@pytest.mark.parametrize(
("provider_name", "agent"),
[
("bedrock", "codex"),
("bedrock", "opencode"),
],
)
def test_incompatible_provider_agent_pairs_fail_before_harbor(provider_name: str, agent: str) -> None:
with pytest.raises(ValueError, match="does not support"):
runner._resolve_agent_runtime_plan(
provider=_provider(provider_name),
agents=[agent],
models={agent: "test-model"},
configured_runtime_env={},
env_mode="docker",
model_sources={agent: "CLI"},
)
def test_openai_mixed_agents_receive_isolated_native_credentials(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
runner.os,
"environ",
{
"PATH": "/usr/bin",
"ANTHROPIC_API_KEY": "anthropic-agent-key",
},
)
plans = runner._resolve_agent_runtime_plan(
provider=_provider("openai"),
agents=["codex", "claude-code"],
models={"codex": "test-model", "claude-code": "anthropic/claude-sonnet-4-5"},
configured_runtime_env={},
env_mode="docker",
model_sources={"codex": "public provider default", "claude-code": "CLI"},
)
assert plans["codex"].staged_env == {
"OPENAI_API_KEY": "${OPENAI_API_KEY}",
"OPENAI_BASE_URL": "${OPENAI_BASE_URL}",
}
assert plans["claude-code"].staged_env == {"ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}"}
assert plans["codex"].provider.provider == "openai"
assert plans["claude-code"].provider.provider == "anthropic"
assert plans["claude-code"].provider.model == "claude-sonnet-4-5"
assert "ANTHROPIC_API_KEY" not in plans["codex"].subprocess_env
assert plans["claude-code"].subprocess_env["ANTHROPIC_API_KEY"] == "anthropic-agent-key"
assert plans["claude-code"].subprocess_env["OPENAI_API_KEY"] == "provider-key"
@pytest.mark.parametrize(
("provider_name", "env_mode", "configured_base_url", "expected_base_url"),
[
("openai", "docker", "https://agent-gateway.example", "https://agent-gateway.example"),
("openai", "e2b", "https://agent-gateway.example/v1", "https://agent-gateway.example"),
(
"openai-compatible",
"docker",
"https://agent-gateway.example/team/v1/",
"https://agent-gateway.example/team",
),
("nv_build", "e2b", "https://agent-gateway.example", "https://agent-gateway.example"),
("nv_build", "e2b", "https://agent-gateway.example/v1", "https://agent-gateway.example"),
(
"nv_build",
"e2b",
"https://agent-gateway.example/team/v1/",
"https://agent-gateway.example/team",
),
],
)
def test_independent_claude_base_url_is_canonical_across_the_runtime_plan(
monkeypatch: pytest.MonkeyPatch,
provider_name: str,
env_mode: str,
configured_base_url: str,
expected_base_url: str,
) -> None:
monkeypatch.setattr(
runner.os,
"environ",
{
"PATH": "/usr/bin",
"ANTHROPIC_API_KEY": "anthropic-agent-key",
"ANTHROPIC_BASE_URL": configured_base_url,
},
)
evaluator_provider = _provider(provider_name)
plan = runner._resolve_agent_runtime_plan(
provider=evaluator_provider,
agents=["claude-code"],
models={"claude-code": "anthropic/claude-sonnet-4-5"},
configured_runtime_env={},
env_mode=env_mode,
model_sources={"claude-code": "CLI"},
)["claude-code"]
assert plan.staged_env == {
"ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}",
"ANTHROPIC_BASE_URL": "${ANTHROPIC_BASE_URL}",
}
assert plan.subprocess_env["ANTHROPIC_API_KEY"] == "anthropic-agent-key"
assert plan.subprocess_env["ANTHROPIC_BASE_URL"] == expected_base_url
assert plan.provider.provider == "anthropic"
assert plan.provider.api_key == "anthropic-agent-key"
assert plan.provider.base_url == expected_base_url
assert plan.provider.base_url + "/v1/messages" == expected_base_url + "/v1/messages"
assert "/v1/v1/" not in plan.provider.base_url + "/v1/messages"
assert evaluator_provider.api_key == "provider-key"
assert evaluator_provider.base_url == "https://provider.example/v1"
if provider_name == "nv_build":
assert plan.subprocess_env["NVIDIA_API_KEY"] == "provider-key"
assert "OPENAI_API_KEY" not in plan.subprocess_env
else:
assert plan.subprocess_env["OPENAI_API_KEY"] == "provider-key"
assert plan.subprocess_env["OPENAI_BASE_URL"] == "https://provider.example/v1"
@pytest.mark.parametrize(
("provider_name", "env_mode"),
[
("openai", "docker"),
("openai-compatible", "docker"),
("nv_build", "e2b"),
],
)
@pytest.mark.parametrize(
"configured_base_url",
[
" ",
"agent-gateway.example/v1",
"https://agent-gateway.example/v1/messages",
"https://url-user:[email protected]/v1",
"https://agent-gateway.example/v1?token=url-secret",
"https://agent-gateway.example/v1\n",
"https://agent-gateway.example/%2576%2531/%256dessages",
],
)
def test_invalid_independent_claude_base_url_fails_before_harbor_without_echoing_secrets(
monkeypatch: pytest.MonkeyPatch,
provider_name: str,
env_mode: str,
configured_base_url: str,
) -> None:
monkeypatch.setattr(
runner.os,
"environ",
{
"PATH": "/usr/bin",
"ANTHROPIC_API_KEY": "anthropic-agent-key",
"ANTHROPIC_BASE_URL": configured_base_url,
},
)
with pytest.raises(ProviderConfigurationError) as exc_info:
runner._resolve_agent_runtime_plan(
provider=_provider(provider_name),
agents=["claude-code"],
models={"claude-code": "anthropic/claude-sonnet-4-5"},
configured_runtime_env={},
env_mode=env_mode,
model_sources={"claude-code": "CLI"},
)
message = str(exc_info.value)
assert "ANTHROPIC_BASE_URL" in message
assert configured_base_url not in message
assert "url-user" not in message
assert "url-secret" not in message
assert "anthropic-agent-key" not in message
@pytest.mark.parametrize("configured_base_url", [None, ""])
def test_unset_independent_claude_base_url_remains_absent(
monkeypatch: pytest.MonkeyPatch,
configured_base_url: str | None,
) -> None:
host_env = {
"PATH": "/usr/bin",
"ANTHROPIC_API_KEY": "anthropic-agent-key",
}
if configured_base_url is not None:
host_env["ANTHROPIC_BASE_URL"] = configured_base_url
monkeypatch.setattr(runner.os, "environ", host_env)
plan = runner._resolve_agent_runtime_plan(
provider=_provider("openai"),
agents=["claude-code"],
models={"claude-code": "anthropic/claude-sonnet-4-5"},
configured_runtime_env={},
env_mode="docker",
model_sources={"claude-code": "CLI"},
)["claude-code"]
assert "ANTHROPIC_BASE_URL" not in plan.staged_env
assert "ANTHROPIC_BASE_URL" not in plan.subprocess_env
assert plan.provider.base_url is None
def test_independent_claude_base_url_optional_normalizer_result_is_not_staged(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
runner.os,
"environ",
{
"PATH": "/usr/bin",
"ANTHROPIC_API_KEY": "anthropic-agent-key",
"ANTHROPIC_BASE_URL": "https://agent-gateway.example/v1",
},
)
def no_configured_base_url(_value: str, *, variable: str) -> None:
assert variable == "ANTHROPIC_BASE_URL"
monkeypatch.setattr(runner, "_normalize_anthropic_base_url", no_configured_base_url)
plan = runner._resolve_agent_runtime_plan(
provider=_provider("openai"),
agents=["claude-code"],
models={"claude-code": "anthropic/claude-sonnet-4-5"},
configured_runtime_env={},
env_mode="docker",
model_sources={"claude-code": "CLI"},
)["claude-code"]
assert "ANTHROPIC_BASE_URL" not in plan.staged_env
assert "ANTHROPIC_BASE_URL" not in plan.subprocess_env
assert plan.provider.base_url is None
@pytest.mark.live
@pytest.mark.skipif(shutil.which("claude") is None, reason="Installed Claude Code CLI is required")
def test_installed_claude_uses_canonical_independent_gateway_from_runtime_plan(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
request_paths: list[str] = []
response_events = [
(
"message_start",
{
"type": "message_start",
"message": {
"id": "msg_test",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5",
"content": [],
"stop_reason": None,
"stop_sequence": None,
"usage": {"input_tokens": 1, "output_tokens": 0},
},
},
),
(
"content_block_start",
{
"type": "content_block_start",
"index": 0,
"content_block": {"type": "text", "text": ""},
},
),
(
"content_block_delta",
{
"type": "content_block_delta",
"index": 0,
"delta": {"type": "text_delta", "text": "gateway-ok"},
},
),
("content_block_stop", {"type": "content_block_stop", "index": 0}),
(
"message_delta",
{
"type": "message_delta",
"delta": {"stop_reason": "end_turn", "stop_sequence": None},
"usage": {"output_tokens": 1},
},
),
("message_stop", {"type": "message_stop"}),
]
response_body = "".join(
f"event: {event}\ndata: {json.dumps(payload, separators=(',', ':'))}\n\n" for event, payload in response_events
).encode()
class RecordingHandler(BaseHTTPRequestHandler):
def log_message(self, _format: str, *args: object) -> None:
del args
def do_POST(self) -> None:
content_length = int(self.headers.get("content-length", "0"))
self.rfile.read(content_length)
request_paths.append(self.path.partition("?")[0])
self.send_response(200)
self.send_header("content-type", "text/event-stream")
self.send_header("content-length", str(len(response_body)))
self.end_headers()
self.wfile.write(response_body)
claude_path = shutil.which("claude")
assert claude_path is not None
with ThreadingHTTPServer(("127.0.0.1", 0), RecordingHandler) as server:
server_thread = threading.Thread(target=server.serve_forever, daemon=True)
server_thread.start()
configured_base_url = f"http://127.0.0.1:{server.server_port}/team/v1"
monkeypatch.setattr(
runner.os,
"environ",
{
"PATH": os.environ["PATH"],
"HOME": str(tmp_path),
"CLAUDE_CONFIG_DIR": str(tmp_path / "claude"),
"ANTHROPIC_API_KEY": "test-agent-key",
"ANTHROPIC_BASE_URL": configured_base_url,
},
)
plan = runner._resolve_agent_runtime_plan(
provider=_provider("openai"),
agents=["claude-code"],
models={"claude-code": "anthropic/claude-sonnet-4-5"},
configured_runtime_env={},
env_mode="docker",
model_sources={"claude-code": "CLI"},
)["claude-code"]
try:
result = subprocess.run(
[
claude_path,
"--bare",
"--print",
"--no-session-persistence",
"--disable-slash-commands",
"--model",
plan.provider.model,
"Reply only with gateway-ok",
],
cwd=tmp_path,
env=dict(plan.subprocess_env),
capture_output=True,
text=True,
timeout=30,
check=False,
)
finally:
server.shutdown()
server_thread.join(timeout=5)
expected_root = f"http://127.0.0.1:{server.server_port}/team"
assert plan.provider.base_url == expected_root
assert plan.subprocess_env["ANTHROPIC_BASE_URL"] == expected_root
assert plan.subprocess_env["ANTHROPIC_API_KEY"] == "test-agent-key"
assert plan.subprocess_env["OPENAI_API_KEY"] == "provider-key"
assert result.returncode == 0, result.stderr
assert result.stdout.strip() == "gateway-ok"
assert request_paths
assert set(request_paths) == {"/team/v1/messages"}
def test_anthropic_mixed_agents_receive_isolated_native_credentials(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
runner.os,
"environ",
{
"PATH": "/usr/bin",
"OPENAI_API_KEY": "openai-agent-key",
"OPENAI_BASE_URL": "https://api.openai.com/v1",
},
)
plans = runner._resolve_agent_runtime_plan(
provider=_provider("anthropic"),
agents=["claude-code", "codex"],
models={"claude-code": "test-model", "codex": "openai/gpt-5.4-mini"},
configured_runtime_env={},
env_mode="docker",
model_sources={"claude-code": "public provider default", "codex": "CLI"},
)
assert plans["claude-code"].staged_env == {
"ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}",
"ANTHROPIC_BASE_URL": "${ANTHROPIC_BASE_URL}",
}
assert plans["codex"].staged_env == {
"OPENAI_API_KEY": "${OPENAI_API_KEY}",
"OPENAI_BASE_URL": "${OPENAI_BASE_URL}",
}
assert plans["claude-code"].provider.provider == "anthropic"
assert plans["codex"].provider.provider == "openai-compatible"
assert plans["codex"].provider.model == "gpt-5.4-mini"
assert "OPENAI_API_KEY" not in plans["claude-code"].subprocess_env
assert plans["codex"].subprocess_env["OPENAI_API_KEY"] == "openai-agent-key"
assert plans["codex"].subprocess_env["ANTHROPIC_API_KEY"] == "provider-key"
def test_anthropic_legacy_v1_base_is_canonical_for_verifier_and_native_agents(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(runner.os, "environ", {"PATH": "/usr/bin"})
provider = resolve_llm_provider(
{
"SKILL_EVAL_LLM_PROVIDER": "anthropic",
"ANTHROPIC_API_KEY": "anthropic-provider-key",
"ANTHROPIC_BASE_URL": "https://gateway.example/team/v1/",
}
)
verifier_env = runner._provider_environment(provider)
plans = runner._resolve_agent_runtime_plan(
provider=provider,
agents=["claude-code", "opencode"],
models={
"claude-code": "claude-sonnet-4-5",
"opencode": "anthropic/claude-sonnet-4-5",
},
configured_runtime_env={},
env_mode="docker",
model_sources={"claude-code": "public provider default", "opencode": "CLI"},
)
assert provider.base_url == "https://gateway.example/team"
assert verifier_env["ANTHROPIC_BASE_URL"] == "https://gateway.example/team"
for agent in ("claude-code", "opencode"):
assert plans[agent].staged_env == {
"ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}",
"ANTHROPIC_BASE_URL": "${ANTHROPIC_BASE_URL}",
}
assert plans[agent].subprocess_env["ANTHROPIC_BASE_URL"] == "https://gateway.example/team"
assert plans[agent].provider.base_url == "https://gateway.example/team"
assert plans[agent].subprocess_env["ANTHROPIC_BASE_URL"] + "/v1/messages" == (
"https://gateway.example/team/v1/messages"
)
assert "/v1/v1/" not in plans[agent].subprocess_env["ANTHROPIC_BASE_URL"] + "/v1/messages"
def test_openai_compatible_codex_stages_selected_provider_pair() -> None:
plans = runner._resolve_agent_runtime_plan(
provider=_provider("openai-compatible"),
agents=["codex"],
models={"codex": "responses-model"},
configured_runtime_env={},
env_mode="docker",
)
assert plans["codex"].staged_env == {
"OPENAI_API_KEY": "${OPENAI_API_KEY}",
"OPENAI_BASE_URL": "${OPENAI_BASE_URL}",
}
def test_local_anthropic_opencode_is_rejected_explicitly() -> None:
with pytest.raises(ValueError, match="local mode"):
runner._resolve_agent_runtime_plan(
provider=_provider("anthropic"),
agents=["opencode"],
models={"opencode": "anthropic/claude-test"},
configured_runtime_env={},
env_mode="local",
)
def test_local_bedrock_agent_is_rejected_explicitly(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "access")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "secret")
with pytest.raises(ValueError, match="local mode"):
runner._resolve_agent_runtime_plan(
provider=_provider("bedrock"),
agents=["claude-code"],
models={"claude-code": "us.anthropic.model"},
configured_runtime_env={},
env_mode="local",
)
def test_docker_bedrock_claude_activates_bedrock_mode(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
runner.os,
"environ",
{
"PATH": "/usr/bin",
"AWS_ACCESS_KEY_ID": "access",
"AWS_SECRET_ACCESS_KEY": "secret",
"AWS_REGION": "us-west-2",
},
)
plans = runner._resolve_agent_runtime_plan(
provider=_provider("bedrock"),
agents=["claude-code"],
models={"claude-code": "us.anthropic.model"},
configured_runtime_env={},
env_mode="docker",
)
plan = plans["claude-code"]
assert plan.staged_env["CLAUDE_CODE_USE_BEDROCK"] == "${CLAUDE_CODE_USE_BEDROCK}"
assert plan.subprocess_env["CLAUDE_CODE_USE_BEDROCK"] == "1"
assert plan.staged_env["AWS_ACCESS_KEY_ID"] == "${AWS_ACCESS_KEY_ID}"
def test_run_harbor_eval_stages_per_agent_credential_trees(
monkeypatch: pytest.MonkeyPatch,
tmp_path,
) -> None:
skill = tmp_path / "demo"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "evals.json").write_text("[]\n", encoding="utf-8")
provider = _provider("nv_build")
emitted: list[tuple[str, bool, dict[str, str], dict[str, str]]] = []
launched: dict[str, tuple[str, str, dict[str, str], dict[str, str]]] = {}
launched_task_selectors: dict[str, list[str]] = {}
collected: dict[str, object] = {}
monkeypatch.setenv("LLM_JUDGE_MODEL", "legacy-judge-model")
monkeypatch.setenv("SKILL_EVAL_JUDGE_MODEL", "judge-model")
monkeypatch.setenv("LLM_JUDGE_FALLBACK_MODELS", "fallback-one,fallback-two")
monkeypatch.setenv("SKILL_EVAL_JUDGE_CHARACTER_FACTS", "1")
monkeypatch.setattr(runner, "resolve_llm_provider", lambda: provider)
monkeypatch.setattr(
runner,
"load_evals_config",
lambda _path: ({"harbor": {"task_source": "evals_json"}}, None),
)
monkeypatch.setattr(runner, "find_evals_file", lambda _path: skill / "evals" / "evals.json")
monkeypatch.setattr(runner, "_check_prerequisites", lambda **_kwargs: [])
def emit(_skill, target, *, with_skill, runtime_env, **_kwargs):
task = target / "case-001"
task.mkdir(parents=True)
(task / "task.toml").write_text(
'schema_version = "1.3"\n\n[task]\nname = "publisher/presentation-name"\n\n'
'[metadata]\nentry_id = "logical-case-001"\n',
encoding="utf-8",
)
emitted.append(
(
str(target.relative_to(tmp_path / "results")),
with_skill,
dict(runtime_env),
dict(_kwargs["verifier_env"]),
)
)
return [task]
def launch(**kwargs):
assert (kwargs["jobs_dir"].parent / "dataset_snapshot.json").is_file()
launched_task_selectors[kwargs["agent"]] = list(kwargs["task_names"])
launched[kwargs["agent"]] = (
str(kwargs["with_skill"].relative_to(tmp_path / "results")),
str(kwargs["baseline"].relative_to(tmp_path / "results")),
dict(kwargs["run_env"]),
dict(kwargs["verifier_env"]),
)
return []
monkeypatch.setattr(runner, "generate_harbor_tasks", emit)
monkeypatch.setattr(runner, "_run_agent_pair", launch)
monkeypatch.setattr(
runtime_preflight,
"probe_model",
lambda selected_provider: runtime_preflight.ModelProbeResult(
True,
selected_provider.provider,
selected_provider.model,
f"model {selected_provider.model} is available",
),
)
monkeypatch.setattr(
runner,
"collect_harbor_results",
lambda **kwargs: (
collected.update(kwargs)
or {
"execution_status": "complete",
"execution_errors": [],
"metrics": list(DEFAULT_METRICS),
"score_policy": LEGACY_SCORE_POLICY,
"attempt_policy": {
"score_policy": LEGACY_SCORE_POLICY,
"score_definition": "historical six-metric mean",
},
"agents": {},
}
),
)
monkeypatch.setattr(runner, "render_agent_eval_html_report", lambda *_args, **_kwargs: tmp_path / "report.html")
result = runner.run_harbor_eval(
skill,
["opencode", "claude-code"],
agent_models={
"opencode": "nvidia/nvidia/nemotron-3-nano-30b-a3b",
"claude-code": "nvidia/nemotron-3.5-lightning-30b-a3b",
},
output_dir=tmp_path / "results",
env_mode="docker",
keep_harbor_jobs=True,
agent_runtime_preflight=False,
)
assert "error" not in result
assert result["score_policy"] == result["attempt_policy"]["score_policy"] == LEGACY_SCORE_POLICY
assert result["attempt_policy"]["score_definition"] == "historical six-metric mean"
assert result["attempt_policy"]["max_attempts"] == 1
persisted = json.loads(Path(result["result_path"]).read_text(encoding="utf-8"))
assert persisted["attempt_policy"]["score_policy"] == LEGACY_SCORE_POLICY
assert {
(path.split("/")[-2], path.split("/")[-1], with_skill)
for path, with_skill, _runtime_env, _verifier_env in emitted
} == {
("opencode", "with", True),
("opencode", "without", False),
("claude-code", "with", True),
("claude-code", "without", False),
}
staged = {
(path.split("/")[-2], path.split("/")[-1]): runtime_env
for path, _with_skill, runtime_env, _verifier_env in emitted
}
assert staged[("opencode", "with")] == {"NVIDIA_API_KEY": "${NVIDIA_API_KEY}"}
assert staged[("claude-code", "with")] == {}
for _path, _with_skill, runtime_env, verifier_env in emitted:
assert "LLM_JUDGE_MODEL" not in runtime_env
assert "SKILL_EVAL_JUDGE_MODEL" not in runtime_env
assert "LLM_JUDGE_FALLBACK_MODELS" not in runtime_env
assert "SKILL_EVAL_JUDGE_CHARACTER_FACTS" not in runtime_env
assert "LLM_JUDGE_MODEL" not in verifier_env
assert "SKILL_EVAL_JUDGE_MODEL" not in verifier_env
assert verifier_env["LLM_JUDGE_FALLBACK_MODELS"] == "${LLM_JUDGE_FALLBACK_MODELS}"
assert verifier_env["SKILL_EVAL_JUDGE_CHARACTER_FACTS"] == "${SKILL_EVAL_JUDGE_CHARACTER_FACTS}"
assert launched["opencode"][0].endswith("_harbor-tasks/opencode/with")
assert launched["claude-code"][1].endswith("_harbor-tasks/claude-code/without")
assert launched_task_selectors == {"opencode": ["case-001"], "claude-code": ["case-001"]}
assert collected["expected_case_ids"] == ["logical-case-001"]
assert collected["case_id_by_task_selector"] == {"case-001": "logical-case-001"}
assert launched["opencode"][2]["LLM_JUDGE_MODEL"] == "legacy-judge-model"
assert launched["opencode"][2]["SKILL_EVAL_JUDGE_MODEL"] == "legacy-judge-model"
assert launched["opencode"][2]["LLM_JUDGE_FALLBACK_MODELS"] == "fallback-one,fallback-two"
assert launched["opencode"][2]["SKILL_EVAL_JUDGE_CHARACTER_FACTS"] == "1"
assert "ANTHROPIC_API_KEY" not in launched["opencode"][2]
assert "OPENAI_API_KEY" not in launched["opencode"][2]
assert launched["claude-code"][2]["NVIDIA_API_KEY"] == "provider-key"
assert launched["opencode"][3] == {
"LLM_JUDGE_FALLBACK_MODELS": "${LLM_JUDGE_FALLBACK_MODELS}",
"SKILL_EVAL_JUDGE_CHARACTER_FACTS": "${SKILL_EVAL_JUDGE_CHARACTER_FACTS}",
"LLM_JUDGE_MODEL": "${LLM_JUDGE_MODEL}",
"SKILL_EVAL_JUDGE_MODEL": "${LLM_JUDGE_MODEL}",
}
assert launched["claude-code"][3] == launched["opencode"][3]
assert result["run_config"]["judge"] == {
"enabled": True,
"provider": "nv_build",
"model": "legacy-judge-model",
"source": "LLM_JUDGE_MODEL",
"override_applied": True,
"reasoning_effort": None,
"character_facts": "1",
"catalog_verification": "degraded",
}
@pytest.mark.parametrize("grading_mode", ["default", "default_plus_custom"])
@pytest.mark.parametrize(
"judge_alias",
["LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL", "LLM_JUDGE_FALLBACK_MODELS", "SKILL_EVAL_JUDGE_CHARACTER_FACTS"],
)
@pytest.mark.parametrize("judge_scope", ["task", "step"])
def test_native_runtime_rejects_task_owned_judge_selection_through_real_staging(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
grading_mode: str,
judge_alias: str,
judge_scope: str,
) -> None:
skill = tmp_path / "demo"
evals = skill / "evals"
native_task = evals / "harbor" / "case-001"
native_task.mkdir(parents=True)
(skill / "SKILL.md").write_text("# Demo\n", encoding="utf-8")
(evals / "evals.json").write_text(
'[{"id":"case-001","question":"Run the native case.","files":[]}]\n',
encoding="utf-8",
)
(evals / "config.yml").write_text(
f"schema_version: 1\nharbor:\n task_source: native_harbor\ngrading:\n mode: {grading_mode}\n",
encoding="utf-8",
)
if grading_mode == "default_plus_custom":
(evals / "grader.py").write_text("def grade(*args, **kwargs):\n return 1\n", encoding="utf-8")
(native_task / "instruction.md").write_text("Run the native case.\n", encoding="utf-8")
authored_judge = f'{judge_alias} = "authored-native-judge"\n'
judge_config = (
f"[verifier]\ntimeout_sec = 180.0\n\n[verifier.env]\n{authored_judge}\n"
if judge_scope == "task"
else f'[[steps]]\nname = "step-one"\n\n[steps.verifier.env]\n{authored_judge}\n'
)
(native_task / "task.toml").write_text(
'schema_version = "1.3"\n\n[task]\nname = "nvidia/case-001"\n\n'
f'[metadata]\nentry_id = "case-001"\n\n{judge_config}[environment]\n',
encoding="utf-8",
)
for name in ("LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL"):
monkeypatch.delenv(name, raising=False)
monkeypatch.setattr(runner, "resolve_llm_provider", lambda: _provider("nv_build"))
monkeypatch.setattr(
runner,
"load_evals_config",
lambda _path: (
{"harbor": {"task_source": "native_harbor"}, "grading": {"mode": grading_mode}},
None,
),
)
monkeypatch.setattr(runner, "_check_prerequisites", lambda **_kwargs: [])
probed_models: list[str] = []
def probe(selected_provider: ProviderConfig) -> runtime_preflight.ModelProbeResult:
probed_models.append(selected_provider.model)
return runtime_preflight.ModelProbeResult(
True,
selected_provider.provider,
selected_provider.model,
f"model {selected_provider.model} is available",
)
staged_tasks: list[Path] = []
def launch(**kwargs):
staged_tasks.append(kwargs["with_skill"] / "case-001")
return []
monkeypatch.setattr(runtime_preflight, "probe_model", probe)
monkeypatch.setattr(runner, "_run_agent_pair", launch)
monkeypatch.setattr(
runner,
"collect_harbor_results",
lambda **_kwargs: {"execution_status": "complete", "execution_errors": [], "metrics": [], "agents": {}},
)
def render_report(_skill_path: Path, output_dir: Path, **_kwargs) -> Path:
report = output_dir / "report.html"
report.write_text("<html></html>\n", encoding="utf-8")
return report
monkeypatch.setattr(runner, "render_agent_eval_html_report", render_report)
result = runner.run_harbor_eval(
skill,
["opencode"],
agent_models={"opencode": "nvidia/nvidia/nemotron-3-nano-30b-a3b"},
output_dir=tmp_path / "results",
env_mode="docker",
grading_mode=grading_mode,
skip_baseline=True,
keep_harbor_jobs=True,
agent_runtime_preflight=False,
)
assert result["execution_status"] == "failed"
assert any("standard grading" in error and "environment control" in error for error in result["error"])
assert result["run_config"]["task_source"] == "native_harbor"
assert staged_tasks == []
def test_custom_only_native_verifier_placeholders_use_task_fallbacks_not_standard_override(
monkeypatch: pytest.MonkeyPatch,
tmp_path,
) -> None:
skill = tmp_path / "demo"
evals = skill / "evals"
native_task = evals / "harbor" / "case-001"
native_task.mkdir(parents=True)
(skill / "SKILL.md").write_text("# Demo\n", encoding="utf-8")
(evals / "evals.json").write_text(
'[{"id":"case-001","question":"Run the native case.","files":[]}]\n',
encoding="utf-8",
)
(evals / "config.yml").write_text(
"schema_version: 1\nharbor:\n task_source: native_harbor\ngrading:\n mode: custom_only\n",
encoding="utf-8",
)
(evals / "grader.py").write_text("def grade(*args, **kwargs):\n return 1\n", encoding="utf-8")
(native_task / "instruction.md").write_text("Run the native case.\n", encoding="utf-8")
(native_task / "task.toml").write_text(
'schema_version = "1.3"\n\n[task]\nname = "nvidia/case-001"\n\n'
'[metadata]\nentry_id = "case-001"\n\n'
'[verifier.env]\nLLM_JUDGE_MODEL = "${LLM_JUDGE_MODEL:-task-legacy-model}"\n'
'SKILL_EVAL_JUDGE_MODEL = "${SKILL_EVAL_JUDGE_MODEL:-task-canonical-model}"\n\n[environment]\n',
encoding="utf-8",
)
launched: dict[str, object] = {}
monkeypatch.setenv("LLM_JUDGE_MODEL", "standard-legacy-model")
monkeypatch.setenv("SKILL_EVAL_JUDGE_MODEL", "standard-canonical-model")
monkeypatch.setenv("LLM_JUDGE_FALLBACK_MODELS", "standard-fallback-one,standard-fallback-two")
monkeypatch.setattr(runner, "resolve_llm_provider", lambda: _provider("nv_build"))
monkeypatch.setattr(
runner,
"load_evals_config",
lambda _path: ({"harbor": {"task_source": "native_harbor"}}, None),
)
monkeypatch.setattr(runner, "_check_prerequisites", lambda **_kwargs: [])
def launch(**kwargs):
launched.update(
with_skill=kwargs["with_skill"],
run_env=dict(kwargs["run_env"]),
verifier_env=dict(kwargs["verifier_env"]),
)
return []
monkeypatch.setattr(runner, "_run_agent_pair", launch)
monkeypatch.setattr(
runtime_preflight,
"probe_model",
lambda selected_provider: runtime_preflight.ModelProbeResult(
True,
selected_provider.provider,
selected_provider.model,
f"model {selected_provider.model} is available",
),
)
monkeypatch.setattr(
runner,
"collect_harbor_results",
lambda **_kwargs: {"execution_status": "complete", "execution_errors": [], "metrics": [], "agents": {}},
)
monkeypatch.setattr(runner, "render_agent_eval_html_report", lambda *_args, **_kwargs: tmp_path / "report.html")
result = runner.run_harbor_eval(
skill,
["opencode"],
agent_models={"opencode": "nvidia/nvidia/nemotron-3-nano-30b-a3b"},
output_dir=tmp_path / "results",
env_mode="docker",
grading_mode="custom_only",
skip_baseline=True,
keep_harbor_jobs=True,
agent_runtime_preflight=False,
)
assert "error" not in result
assert result["run_config"]["judge"] == {"enabled": False}
assert launched["verifier_env"] == {}
task_config = tomllib.loads(
(launched["with_skill"] / "case-001" / "task.toml").read_text(encoding="utf-8") # type: ignore[operator]
)
authored_env = task_config["verifier"]["env"]
run_env = launched["run_env"]
assert "LLM_JUDGE_FALLBACK_MODELS" not in authored_env
assert "LLM_JUDGE_FALLBACK_MODELS" not in run_env
task_fallbacks = {
"LLM_JUDGE_MODEL": "task-legacy-model",
"SKILL_EVAL_JUDGE_MODEL": "task-canonical-model",
}
for name, fallback in task_fallbacks.items():
assert name not in run_env
assert authored_env[name] == f"${{{name}:-{fallback}}}"
with monkeypatch.context() as harbor_process:
harbor_process.setattr(os, "environ", dict(run_env))
assert resolve_env_vars({name: authored_env[name]}) == {name: fallback}
def test_run_harbor_eval_rejects_provenance_key_inside_skill_before_creation(
monkeypatch: pytest.MonkeyPatch,
tmp_path,
) -> None:
skill = tmp_path / "demo"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "evals.json").write_text("[]\n", encoding="utf-8")
key_path = skill / "private-output-provenance.key"
monkeypatch.setenv("SKILLEVALUATOR_OUTPUT_PROVENANCE_KEY_FILE", str(key_path))
monkeypatch.setattr(runner, "resolve_llm_provider", lambda: _provider("nv_build"))
monkeypatch.setattr(
runner,
"load_evals_config",
lambda _path: ({"harbor": {"task_source": "evals_json"}}, None),
)
monkeypatch.setattr(runner, "find_evals_file", lambda _path: skill / "evals" / "evals.json")
monkeypatch.setattr(runner, "_check_prerequisites", lambda **_kwargs: [])
monkeypatch.setattr(
runtime_preflight,
"probe_model",
lambda selected_provider: runtime_preflight.ModelProbeResult(
True,
selected_provider.provider,
selected_provider.model,
f"model {selected_provider.model} is available",
),
)
result = runner.run_harbor_eval(
skill,
["opencode"],
output_dir=tmp_path / "results",
env_mode="docker",
agent_runtime_preflight=False,
)
assert "error" in result
assert "provenance key must be outside" in result["error"][0]
assert not key_path.exists()
assert not (tmp_path / "results").exists()
def test_harbor_subprocess_environment_excludes_arbitrary_host_secrets(
monkeypatch: pytest.MonkeyPatch,
tmp_path,
) -> None:
host_environment = {
"PATH": os.environ["PATH"],
"HOME": str(tmp_path),
"USERPROFILE": str(tmp_path),
"TMPDIR": str(tmp_path / "tmp"),
"LANG": "C.UTF-8",
"DOCKER_HOST": "unix:///safe/docker.sock",
"HOST_DATABASE_PASSWORD": "must-not-leak",
"SSH_AUTH_SOCK": "/private/agent.sock",
"ANTHROPIC_API_KEY": "unrelated-provider-secret",
"E2B_API_KEY": "unselected-backend-secret",
}
monkeypatch.setattr(runner.os, "environ", host_environment)
provider = _provider()
provider_env = runner._provider_environment(provider)
environment = runner._harbor_subprocess_environment(
env_mode="docker",
provider=provider,
configured_runtime_env={"SERVICE_TOKEN": "declared-runtime-secret"},
provider_env=provider_env,
)
assert environment == {
"PATH": host_environment["PATH"],
"HOME": str(tmp_path),
"TMPDIR": str(tmp_path / "tmp"),
"LANG": "C.UTF-8",
"DOCKER_HOST": "unix:///safe/docker.sock",
"SSH_AUTH_SOCK": "/private/agent.sock",
"SERVICE_TOKEN": "declared-runtime-secret",
"USERPROFILE": str(tmp_path),
**provider_env,
}
def test_harbor_subprocess_environment_includes_only_selected_backend_credentials(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
runner.os,
"environ",
{
"PATH": "/usr/bin",
"HOME": "/home/test",
"E2B_API_KEY": "selected-key",
"MODAL_TOKEN_ID": "unselected-id",
"MODAL_TOKEN_SECRET": "unselected-secret",
},
)
provider = _provider()
environment = runner._harbor_subprocess_environment(
env_mode="e2b",
provider=provider,
configured_runtime_env={},
provider_env=runner._provider_environment(provider),
)
assert environment["E2B_API_KEY"] == "selected-key"
assert "MODAL_TOKEN_ID" not in environment
assert "MODAL_TOKEN_SECRET" not in environment
def test_harbor_subprocess_environment_preserves_trusted_proxy_and_tls_controls(
monkeypatch: pytest.MonkeyPatch,
) -> None:
network_environment = {
"HTTP_PROXY": "http://proxy.example:8080",
"HTTPS_PROXY": "https://proxy.example:8443",
"NO_PROXY": "localhost,127.0.0.1",
"ALL_PROXY": "socks5://proxy.example:1080",
"http_proxy": "http://legacy-proxy.example:8080",
"https_proxy": "https://legacy-proxy.example:8443",
"no_proxy": "metadata.internal",
"all_proxy": "socks5://legacy-proxy.example:1080",
"SSL_CERT_FILE": "/etc/ssl/custom.pem",
"SSL_CERT_DIR": "/etc/ssl/custom",
"REQUESTS_CA_BUNDLE": "/etc/ssl/requests.pem",
"CURL_CA_BUNDLE": "/etc/ssl/curl.pem",
}
monkeypatch.setattr(
runner.os,
"environ",
{"PATH": "/usr/bin", "HOME": "/home/test", **network_environment, "UNRELATED_SECRET": "no"},
)
provider = _provider()
environment = runner._harbor_subprocess_environment(
env_mode="e2b",
provider=provider,
configured_runtime_env={},
provider_env=runner._provider_environment(provider),
)
assert {name: environment[name] for name in network_environment} == network_environment
assert "UNRELATED_SECRET" not in environment
def test_harbor_backend_environment_allowlist_covers_every_native_022_mode() -> None:
expected = {
"docker": {
"COMPOSE_ANSI",
"COMPOSE_HTTP_TIMEOUT",
"COMPOSE_IGNORE_ORPHANS",
"COMPOSE_PARALLEL_LIMIT",
"COMPOSE_PROGRESS",
"COMPOSE_STATUS_STDOUT",
"DOCKER_API_VERSION",
"DOCKER_AUTH_CONFIG",
"DOCKER_CERT_PATH",
"DOCKER_CONFIG",
"DOCKER_CONTEXT",
"DOCKER_CUSTOM_HEADERS",
"DOCKER_DEFAULT_PLATFORM",
"DOCKER_HOST",
"DOCKER_TLS",
"DOCKER_TLS_VERIFY",
"SSH_AUTH_SOCK",
},
"daytona": {
"DAYTONA_API_KEY",
"DAYTONA_API_URL",
"DAYTONA_HAPPY_EYEBALLS_DELAY",
"DAYTONA_JWT_TOKEN",
"DAYTONA_ORGANIZATION_ID",
"DAYTONA_SERVER_URL",
"DAYTONA_TARGET",
},
"e2b": {"E2B_API_KEY", "E2B_API_URL", "E2B_DOMAIN", "E2B_SANDBOX_URL"},
"modal": {
"MODAL_CONFIG_PATH",
"MODAL_ENVIRONMENT",
"MODAL_OVERRIDE_HEADERS",
"MODAL_PROFILE",
"MODAL_SERVER_URL",
"MODAL_TOKEN_ID",
"MODAL_TOKEN_SECRET",
},
"runloop": {"RUNLOOP_API_KEY", "RUNLOOP_BASE_URL", "RUNLOOP_CUSTOM_HEADERS"},
"langsmith": {
"LANGCHAIN_API_KEY",
"LANGCHAIN_ENDPOINT",
"LANGSMITH_API_KEY",
"LANGSMITH_CONFIG_FILE",
"LANGSMITH_ENDPOINT",
"LANGSMITH_PROFILE",
"LANGSMITH_SANDBOX_API_URL",
"LANGSMITH_WORKSPACE_ID",
},
"ec2": {
"AWS_ACCOUNT_ID",
"AWS_ACCOUNT_ID_ENDPOINT_MODE",
"AWS_ACCESS_KEY_ID",
"AWS_AUTH_SCHEME_PREFERENCE",
"AWS_CA_BUNDLE",
"AWS_CONFIG_FILE",
"AWS_CONTAINER_AUTHORIZATION_TOKEN",
"AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
"AWS_CONTAINER_CREDENTIALS_FULL_URI",
"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI",
"AWS_CREDENTIAL_EXPIRATION",
"AWS_CREDENTIAL_FILE",
"AWS_CSM_CLIENT_ID",
"AWS_CSM_ENABLED",
"AWS_CSM_HOST",
"AWS_CSM_PORT",
"AWS_DATA_PATH",
"AWS_DEFAULT_PROFILE",
"AWS_DEFAULT_REGION",
"AWS_DEFAULTS_MODE",
"AWS_DISABLE_HOST_PREFIX_INJECTION",
"AWS_DISABLE_REQUEST_COMPRESSION",
"AWS_EC2_METADATA_DISABLED",
"AWS_EC2_METADATA_SERVICE_ENDPOINT",
"AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE",
"AWS_EC2_METADATA_V1_DISABLED",
"AWS_ENDPOINT_DISCOVERY_ENABLED",
"AWS_ENDPOINT_URL",
"AWS_ENDPOINT_URL_EC2",
"AWS_ENDPOINT_URL_SIGNIN",
"AWS_ENDPOINT_URL_SSO",
"AWS_ENDPOINT_URL_SSO_OIDC",
"AWS_ENDPOINT_URL_STS",
"AWS_EXECUTION_ENV",
"AWS_IGNORE_CONFIGURED_ENDPOINT_URLS",
"AWS_IMDS_USE_IPV6",
"AWS_LOGIN_CACHE_DIRECTORY",
"AWS_MAX_ATTEMPTS",
"AWS_METADATA_SERVICE_NUM_ATTEMPTS",
"AWS_METADATA_SERVICE_TIMEOUT",
"AWS_NEW_RETRIES_2026",
"AWS_PROFILE",
"AWS_REGION",
"AWS_RETRY_MODE",
"AWS_REQUEST_CHECKSUM_CALCULATION",
"AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES",
"AWS_RESPONSE_CHECKSUM_VALIDATION",
"AWS_ROLE_ARN",
"AWS_ROLE_SESSION_NAME",
"AWS_SDK_LOAD_CONFIG",
"AWS_SDK_UA_APP_ID",
"AWS_SECRET_ACCESS_KEY",
"AWS_SECURITY_TOKEN",
"AWS_SESSION_TOKEN",
"AWS_SHARED_CREDENTIALS_FILE",
"AWS_SIGV4A_SIGNING_REGION_SET",
"AWS_STS_REGIONAL_ENDPOINTS",
"AWS_USE_DUALSTACK_ENDPOINT",
"AWS_USE_FIPS_ENDPOINT",
"AWS_WEB_IDENTITY_TOKEN_FILE",
"BOTOCORE_TCP_KEEPALIVE",
"SSH_AUTH_SOCK",
},
"gke": {
"CLOUDSDK_ACTIVE_CONFIG_NAME",
"CLOUDSDK_AUTH_ACCESS_TOKEN",
"CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE",
"CLOUDSDK_AUTH_IMPERSONATE_SERVICE_ACCOUNT",
"CLOUDSDK_CONFIG",
"CLOUDSDK_CORE_ACCOUNT",
"CLOUDSDK_CORE_PROJECT",
"GCP_PROJECT",
"GOOGLE_APPLICATION_CREDENTIALS",
"GOOGLE_CLOUD_PROJECT",
"GOOGLE_CLOUD_QUOTA_PROJECT",
"KUBECONFIG",
"SKILLEVALUATOR_GKE_ALLOW_WORKLOAD_IDENTITY",
"SKILLEVALUATOR_GKE_AUTOPILOT",
"SKILLEVALUATOR_GKE_METADATA_PROBE_IMAGE",
},
"ack": {
"COMPOSE_ANSI",
"COMPOSE_HTTP_TIMEOUT",
"COMPOSE_IGNORE_ORPHANS",
"COMPOSE_PARALLEL_LIMIT",
"COMPOSE_PROGRESS",
"COMPOSE_STATUS_STDOUT",
"DOCKER_API_VERSION",
"DOCKER_AUTH_CONFIG",
"DOCKER_CERT_PATH",
"DOCKER_CONFIG",
"DOCKER_CONTEXT",
"DOCKER_CUSTOM_HEADERS",
"DOCKER_DEFAULT_PLATFORM",
"DOCKER_HOST",
"DOCKER_TLS",
"DOCKER_TLS_VERIFY",
"KUBECONFIG",
"KUBERNETES_SERVICE_HOST",
"KUBERNETES_SERVICE_PORT",
"SSH_AUTH_SOCK",
},
"openshift": {"KUBECONFIG"},
"novita": {
"NOVITA_ACCESS_TOKEN",
"NOVITA_API_KEY",
"NOVITA_API_URL",
"NOVITA_BASE_URL",
"NOVITA_DOMAIN",
"NOVITA_SANDBOX_URL",
},
"apple-container": set(),
"singularity": {
"APPTAINER_AUTHFILE",
"APPTAINER_CONFIGDIR",
"APPTAINER_DOCKER_PASSWORD",
"APPTAINER_DOCKER_USERNAME",
"SINGULARITY_AUTHFILE",
"SINGULARITY_CONFIGDIR",
"SINGULARITY_DOCKER_PASSWORD",
"SINGULARITY_DOCKER_USERNAME",
},
"islo": {"ISLO_API_KEY", "ISLO_API_URL", "ISLO_COMPUTE_URL"},
"tensorlake": {
"TENSORLAKE_API_KEY",
"TENSORLAKE_API_URL",
"TENSORLAKE_ORGANIZATION_ID",
"TENSORLAKE_PAT",
"TENSORLAKE_PROJECT_ID",
"TENSORLAKE_SANDBOX_PROXY_URL",
},
"cwsandbox": {"CWSANDBOX_API_KEY", "CWSANDBOX_BASE_URL"},
"wandb": {"NETRC", "WANDB_API_KEY", "WANDB_BASE_URL", "WANDB_ENTITY", "WANDB_PROJECT"},
"use-computer": {
"USE_COMPUTER_API_KEY",
"USE_COMPUTER_HOST",
"USE_COMPUTER_SNAPSHOT",
"USE_COMPUTER_VERSION",
},
"blaxel": {
"BL_API_KEY",
"BL_API_VERSION",
"BL_CLIENT_CREDENTIALS",
"BL_ENV",
"BL_REGION",
"BL_WORKSPACE",
},
"beam": {
"API_HOST",
"API_PORT",
"BEAM_TOKEN",
"GATEWAY_HOST",
"GATEWAY_PORT",
"INTERNAL_API_HOST",
"INTERNAL_API_PORT",
"REALTIME_HOST",
},
"skypilot": {
"COMPOSE_ANSI",
"COMPOSE_HTTP_TIMEOUT",
"COMPOSE_IGNORE_ORPHANS",
"COMPOSE_PARALLEL_LIMIT",
"COMPOSE_PROGRESS",
"COMPOSE_STATUS_STDOUT",
"DOCKER_API_VERSION",
"DOCKER_AUTH_CONFIG",
"DOCKER_CERT_PATH",
"DOCKER_CONFIG",
"DOCKER_CONTEXT",
"DOCKER_CUSTOM_HEADERS",
"DOCKER_DEFAULT_PLATFORM",
"DOCKER_HOST",
"DOCKER_TLS",
"DOCKER_TLS_VERIFY",
"HARBOR_SKYPILOT_REGISTRY",
"SKYPILOT_API_SERVER_ENDPOINT",
"SKYPILOT_GLOBAL_CONFIG",
"SKYPILOT_PROJECT_CONFIG",
"SKYPILOT_SERVICE_ACCOUNT_TOKEN",
"SSH_AUTH_SOCK",
},
"hyperbrowser": {
"COMPOSE_ANSI",
"COMPOSE_HTTP_TIMEOUT",
"COMPOSE_IGNORE_ORPHANS",
"COMPOSE_PARALLEL_LIMIT",
"COMPOSE_PROGRESS",
"COMPOSE_STATUS_STDOUT",
"DOCKER_API_VERSION",
"DOCKER_AUTH_CONFIG",
"DOCKER_CERT_PATH",
"DOCKER_CONFIG",
"DOCKER_CONTEXT",
"DOCKER_CUSTOM_HEADERS",
"DOCKER_DEFAULT_PLATFORM",
"DOCKER_HOST",
"DOCKER_TLS",
"DOCKER_TLS_VERIFY",
"HYPERBROWSER_API_KEY",
"HYPERBROWSER_BASE_URL",
"SSH_AUTH_SOCK",
},
"vercel": {"VERCEL_OIDC_TOKEN", "VERCEL_PROJECT_ID", "VERCEL_TEAM_ID", "VERCEL_TOKEN"},
"runta": {"RUNTA_CONFIG", "RUNTA_ENDPOINT", "RUNTA_TOKEN"},
"mosaic": {
"MAR_API_TOKEN",
"MAR_CONFIG",
"MAR_ENDPOINT",
"MOSAIC_API_TOKEN",
"MOSAIC_API_URL",
"MOSAIC_CONFIG",
"MOSAIC_REGISTRY_PASSWORD",
"MOSAIC_REGISTRY_USERNAME",
"MOSAIC_RETRIES",
},
}
# Kata runs on the host's Docker engine, so it selects exactly Docker's variables.
expected["kata"] = expected["docker"]
assert set(expected) == HARBOR_NATIVE_ENV_MODES
assert {mode: set(names) for mode, names in runner._HARBOR_ENV_MODE_VARS.items()} == expected
def test_direct_run_returns_configuration_error_for_cyclic_environment_kwargs(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
cyclic: dict[str, object] = {}
cyclic["nested"] = cyclic
monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
monkeypatch.setattr(runner, "load_evals_config", lambda _path: ({}, None))
result = runner._run_harbor_eval_impl(
tmp_path,
["opencode"],
environment_kwargs={"options": cyclic},
_evaluator_skill_path=tmp_path,
)
assert result == {"error": ["Invalid --environment-kwarg: must not contain cyclic values"]}
@pytest.mark.parametrize("timeout_multiplier", [float("nan"), float("inf"), float("-inf")])
def test_direct_run_rejects_nonfinite_timeout_multiplier_before_creating_results(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
timeout_multiplier: float,
) -> None:
monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
monkeypatch.setattr(
runner,
"load_evals_config",
lambda _path: ({"harbor": {"task_source": "evals_json"}}, None),
)
result = runner._run_harbor_eval_impl(
tmp_path,
["opencode"],
timeout_multiplier=timeout_multiplier,
output_dir=tmp_path / "results",
_evaluator_skill_path=tmp_path,
)
assert result == {"error": ["timeout_multiplier must be a finite number greater than 0"]}
assert not (tmp_path / "results").exists()
def test_direct_run_rejects_overflowing_timeout_multiplier_before_creating_results(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
monkeypatch.setattr(
runner,
"load_evals_config",
lambda _path: ({"harbor": {"task_source": "evals_json"}}, None),
)
result = runner._run_harbor_eval_impl(
tmp_path,
["opencode"],
timeout_multiplier=10**1000,
output_dir=tmp_path / "results",
_evaluator_skill_path=tmp_path,
)
assert result == {"error": ["timeout_multiplier must be a finite number greater than 0"]}
assert not (tmp_path / "results").exists()
def test_direct_run_rejects_finite_multiplier_that_overflows_default_timeouts(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
monkeypatch.setattr(
runner,
"load_evals_config",
lambda _path: ({"harbor": {"task_source": "evals_json"}}, None),
)
result = runner._run_harbor_eval_impl(
tmp_path,
["opencode"],
timeout_multiplier=1e308,
output_dir=tmp_path / "results",
_evaluator_skill_path=tmp_path,
)
assert result == {"error": ["timeout_multiplier must yield finite Harbor timeouts"]}
assert not (tmp_path / "results").exists()
def test_direct_run_rejects_overflowing_pass_threshold_before_creating_results(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
monkeypatch.setattr(
runner,
"load_evals_config",
lambda _path: ({"harbor": {"task_source": "evals_json"}}, None),
)
result = runner._run_harbor_eval_impl(
tmp_path,
["opencode"],
pass_threshold=10**1000,
output_dir=tmp_path / "results",
_evaluator_skill_path=tmp_path,
)
assert result == {"error": ["pass_threshold must be a finite number between 0.0 and 1.0"]}
assert not (tmp_path / "results").exists()
@pytest.mark.parametrize("setup_key", ["pre_agent_setup", "setup_commands"])
def test_paired_run_rejects_normalized_skill_owned_setup_before_environment_preflight(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
setup_key: str,
) -> None:
skill = tmp_path / "skill"
evals = skill / "evals"
evals.mkdir(parents=True)
(evals / "config.yml").write_text(
f"schema_version: 1\nharbor:\n {setup_key}: echo ready\n",
encoding="utf-8",
)
monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
def unexpected_preflight(**_kwargs: object) -> list[str]:
pytest.fail("paired skill-owned setup reached environment preflight")
monkeypatch.setattr(runner, "_check_prerequisites", unexpected_preflight)
result = runner._run_harbor_eval_impl(
skill,
["opencode"],
output_dir=tmp_path / "results",
_evaluator_skill_path=skill,
)
assert result == {
"error": [
"harbor.pre_agent_setup/setup_commands cannot run in a paired evaluation; "
"use --skip-baseline for a with-skill-only run"
]
}
assert not (tmp_path / "results").exists()
@pytest.mark.parametrize("setup_key", ["pre_agent_setup", "setup_commands"])
def test_skip_baseline_keeps_normalized_skill_owned_setup_enabled(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
setup_key: str,
) -> None:
skill = tmp_path / "skill"
evals = skill / "evals"
evals.mkdir(parents=True)
(evals / "config.yml").write_text(
f"schema_version: 1\nharbor:\n {setup_key}: echo ready\n",
encoding="utf-8",
)
monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
monkeypatch.setattr(runner, "_check_prerequisites", lambda **_kwargs: ["preflight sentinel"])
result = runner._run_harbor_eval_impl(
skill,
["opencode"],
skip_baseline=True,
output_dir=tmp_path / "results",
_evaluator_skill_path=skill,
)
assert result == {"error": ["preflight sentinel"]}
@pytest.mark.parametrize(
("env_mode", "selected_name"),
[
("ec2", "AWS_ACCESS_KEY_ID"),
("ec2", "SSH_AUTH_SOCK"),
("docker", "SSH_AUTH_SOCK"),
("modal", "MODAL_SERVER_URL"),
("gke", "CLOUDSDK_ACTIVE_CONFIG_NAME"),
("gke", "CLOUDSDK_CORE_PROJECT"),
("ack", "KUBECONFIG"),
("ack", "KUBERNETES_SERVICE_HOST"),
("ack", "KUBERNETES_SERVICE_PORT"),
("ack", "DOCKER_CONFIG"),
("openshift", "KUBECONFIG"),
("novita", "NOVITA_SANDBOX_URL"),
("singularity", "APPTAINER_AUTHFILE"),
("tensorlake", "TENSORLAKE_ORGANIZATION_ID"),
("blaxel", "BL_API_KEY"),
("beam", "BEAM_TOKEN"),
("beam", "API_HOST"),
("skypilot", "SKYPILOT_API_SERVER_ENDPOINT"),
("skypilot", "DOCKER_CONFIG"),
("hyperbrowser", "HYPERBROWSER_API_KEY"),
("hyperbrowser", "DOCKER_HOST"),
("wandb", "NETRC"),
("vercel", "VERCEL_OIDC_TOKEN"),
("runta", "RUNTA_TOKEN"),
("runta", "RUNTA_ENDPOINT"),
("mosaic", "MOSAIC_API_TOKEN"),
("mosaic", "MOSAIC_API_URL"),
("mosaic", "MOSAIC_CONFIG"),
("mosaic", "MAR_API_TOKEN"),
],
)
def test_new_harbor_022_backend_environment_is_selected_without_cross_backend_leakage(
monkeypatch: pytest.MonkeyPatch,
env_mode: str,
selected_name: str,
) -> None:
host_environment = {
"PATH": "/usr/bin",
"HOME": "/home/test",
selected_name: "selected-value",
"E2B_API_KEY": "unselected-value",
}
monkeypatch.setattr(runner.os, "environ", host_environment)
provider = _provider()
environment = runner._harbor_subprocess_environment(
env_mode=env_mode,
provider=provider,
configured_runtime_env={},
provider_env=runner._provider_environment(provider),
)
assert environment[selected_name] == "selected-value"
assert "E2B_API_KEY" not in environment
def test_daytona_subprocess_environment_preserves_jwt_auth_pair(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
runner.os,
"environ",
{
"PATH": "/usr/bin",
"HOME": "/home/test",
"DAYTONA_JWT_TOKEN": "jwt-token",
"DAYTONA_ORGANIZATION_ID": "organization-id",
},
)
provider = _provider()
environment = runner._harbor_subprocess_environment(
env_mode="daytona",
provider=provider,
configured_runtime_env={},
provider_env=runner._provider_environment(provider),
)
assert environment["DAYTONA_JWT_TOKEN"] == "jwt-token"
assert environment["DAYTONA_ORGANIZATION_ID"] == "organization-id"
def test_config_declared_substitution_is_resolved_without_leaking_source_variable(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
runner.os,
"environ",
{"PATH": "/usr/bin", "HOME": "/home/test", "HOST_AGENT_LABEL": "runtime-value", "UNRELATED_SECRET": "no"},
)
configured_runtime_env, errors = runner._resolve_runtime_env(
{"AGENT_LABEL": "${HOST_AGENT_LABEL}"},
env_mode="docker",
)
provider = _provider()
environment = runner._harbor_subprocess_environment(
env_mode="docker",
provider=provider,
configured_runtime_env=configured_runtime_env,
provider_env=runner._provider_environment(provider),
)
assert errors == []
assert environment["AGENT_LABEL"] == "runtime-value"
assert "HOST_AGENT_LABEL" not in environment
assert "UNRELATED_SECRET" not in environment
def test_runtime_env_rejects_host_process_control_names() -> None:
unsafe_names = (
"PATH",
"PYTHONPATH",
"HOME",
"USERPROFILE",
"XDG_CONFIG_HOME",
"CLAUDE_CONFIG_DIR",
"CLAUDE_CODE_DISABLE_POLICY_SKILLS",
"CODEX_HOME",
"GEMINI_CLI_HOME",
"OPENCODE_CONFIG_DIR",
"LD_PRELOAD",
"DYLD_INSERT_LIBRARIES",
"BASH_ENV",
"BASH_FUNC_hidden%%",
"NODE_OPTIONS",
"LLM_JUDGE_FALLBACK_MODELS",
"DOCKER_HOST",
"COMPOSE_FILE",
"HARBOR_HOME",
"OTEL_EXPORTER_OTLP_ENDPOINT",
"HTTPS_PROXY",
"E2B_API_KEY",
"AWS_CONFIG_FILE",
"AWS_ENDPOINT_URL",
"AWS_PROFILE",
"AWS_WEB_IDENTITY_TOKEN_FILE",
)
for name in unsafe_names:
resolved, errors = runner._resolve_runtime_env(
{name: "attacker-controlled"},
env_mode="docker",
)
assert name not in resolved
assert any(name in error and "host process" in error for error in errors)
def test_bedrock_subprocess_environment_keeps_only_explicit_aws_provider_values(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
runner.os,
"environ",
{
"PATH": "/usr/bin",
"HOME": "/home/test",
"AWS_ACCESS_KEY_ID": "access-key",
"AWS_SECRET_ACCESS_KEY": "secret-key",
"AWS_SESSION_TOKEN": "session-token",
"AWS_PROFILE": "evaluation",
"AWS_BEARER_TOKEN_BEDROCK": "bearer-token",
"AWS_ROLE_ARN": "arn:aws:iam::123456789012:role/evaluator",
"AWS_ROLE_SESSION_NAME": "evaluation-session",
"AWS_WEB_IDENTITY_TOKEN_FILE": "/var/run/secrets/token",
"OTHER_CLOUD_SECRET": "must-not-leak",
},
)
provider = _provider("bedrock")
provider_env = runner._provider_environment(provider)
environment = runner._harbor_subprocess_environment(
env_mode="docker",
provider=provider,
configured_runtime_env={},
provider_env=provider_env,
)
assert provider_env["AWS_ACCESS_KEY_ID"] == "access-key"
assert provider_env["AWS_BEARER_TOKEN_BEDROCK"] == "bearer-token"
assert provider_env["AWS_WEB_IDENTITY_TOKEN_FILE"] == "/var/run/secrets/token"
assert {
"AWS_ACCESS_KEY_ID",
"AWS_BEARER_TOKEN_BEDROCK",
"AWS_WEB_IDENTITY_TOKEN_FILE",
}.issubset(_verifier_env_vars(provider_env))
assert environment["AWS_ACCESS_KEY_ID"] == "access-key"
assert environment["AWS_SECRET_ACCESS_KEY"] == "secret-key"
assert environment["AWS_SESSION_TOKEN"] == "session-token"
assert environment["AWS_PROFILE"] == "evaluation"
assert environment["AWS_BEARER_TOKEN_BEDROCK"] == "bearer-token"
assert environment["AWS_ROLE_ARN"].endswith(":role/evaluator")
assert environment["AWS_ROLE_SESSION_NAME"] == "evaluation-session"
assert environment["AWS_WEB_IDENTITY_TOKEN_FILE"] == "/var/run/secrets/token"
assert "OTHER_CLOUD_SECRET" not in environment
def test_bedrock_host_credential_allowlist_is_staged_for_the_verifier() -> None:
expected = set(runner._BEDROCK_HOST_ENV_VARS) | {"AWS_REGION"}
assert set(_verifier_env_vars(dict.fromkeys(expected, "configured"))) == expected
def test_local_nvidia_provider_preserves_explicit_codex_credentials(
monkeypatch: pytest.MonkeyPatch,
tmp_path,
) -> None:
monkeypatch.setattr(
runner.os,
"environ",
{
"PATH": "/usr/bin",
"HOME": str(tmp_path),
"SKILLEVALUATOR_RUNTIME_DIR": str(tmp_path / "runtimes"),
},
)
provider = _provider("nv_build")
environment = runner._harbor_subprocess_environment(
env_mode="local",
provider=provider,
configured_runtime_env={
"OPENAI_API_KEY": "codex-key",
"OPENAI_BASE_URL": "https://api.openai.com/v1",
},
provider_env=runner._provider_environment(provider),
)
assert environment["NVIDIA_API_KEY"] == "provider-key"
assert environment["OPENAI_API_KEY"] == "codex-key"
assert environment["OPENAI_BASE_URL"] == "https://api.openai.com/v1"
def test_resolve_agent_runtime_plan_preserves_refreshed_adc_token(monkeypatch: pytest.MonkeyPatch) -> None:
"""Verify _resolve_agent_runtime_plan preserves refreshed ADC token and does not overwrite with expired token."""
base_url = "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi"
provider = ProviderConfig(
provider="openai",
model="google/gemini-3.8-flash",
api_key="expired-token",
base_url=base_url,
litellm_model="openai/google/gemini-3.8-flash",
credential_env="ADC",
base_url_env="OPENAI_BASE_URL",
)
monkeypatch.setattr(runner, "_get_google_access_token", lambda **_kw: "fresh-token")
monkeypatch.setattr(runner.os, "environ", {"PATH": "/usr/bin"})
plans = runner._resolve_agent_runtime_plan(
provider=provider,
agents=["opencode", "codex"],
models={
"opencode": "openai/google/gemini-3.8-flash",
"codex": "google/gemini-3.8-flash",
},
configured_runtime_env={},
env_mode="docker",
)
for agent in ("opencode", "codex"):
plan = plans[agent]
assert plan.subprocess_env["OPENAI_API_KEY"] == "fresh-token"
assert plan.provider.api_key == "fresh-token"
assert plan.subprocess_env["OPENAI_BASE_URL"] == base_url
def test_run_harbor_reissues_adc_token_for_bounded_jobs(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""Verify _run_harbor refreshes ADC token in child env without leaking secrets to argv and sets a bounded timeout."""
base_url = "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi"
captured_command: list[str] = []
captured_env: dict[str, str] = {}
captured_timeout: float | None = None
def mock_run(command, *args, **kwargs):
nonlocal captured_timeout
captured_command.extend(command)
captured_env.update(kwargs.get("env") or {})
captured_timeout = kwargs.get("timeout_seconds")
return runner._BoundedHarborProcessResult(returncode=0, output_tail="", output_exceeded=False)
monkeypatch.setattr(runner, "_run_bounded_harbor_process", mock_run)
monkeypatch.setattr(runner, "_validate_harbor_job_result", lambda *_args, **_kwargs: (True, "success"))
monkeypatch.setattr(runner, "_get_google_access_token", lambda **_kw: "fresh-reissued-token")
run_env = {
"OPENAI_API_KEY": "stale-initial-token",
"OPENAI_BASE_URL": base_url,
"SKILL_EVAL_LLM_CREDENTIAL_SOURCE": "ADC",
"PATH": "/usr/bin",
}
verifier_env = {"LLM_JUDGE_MODEL": "${LLM_JUDGE_MODEL}"}
ok, _detail = runner._run_harbor(
dataset=tmp_path / "dataset",
agent="opencode",
job_name="test-job",
env_mode="docker",
model="google/gemini-3.8-flash",
jobs_dir=tmp_path / "jobs",
run_env=run_env,
n_attempts=1,
n_concurrent=1,
timeout_multiplier=1.0,
override_cpus=None,
override_memory_mb=None,
override_storage_mb=None,
verifier_env=verifier_env,
)
assert ok is True
rendered_argv = " ".join(captured_command)
assert "fresh-reissued-token" not in rendered_argv
assert "stale-initial-token" not in rendered_argv
assert "OPENAI_API_KEY=" not in rendered_argv
assert captured_env.get("OPENAI_API_KEY") == "fresh-reissued-token"
assert captured_timeout is not None and 0 < captured_timeout < 3600
def test_run_harbor_preserves_explicit_vertex_credential(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
"""Preserve explicit operator credentials on Vertex URLs without overwriting from host ADC."""
base_url = "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi"
captured_env: dict[str, str] = {}
captured_timeout: float | None = None
def mock_run(command, *args, **kwargs):
nonlocal captured_timeout
captured_env.update(kwargs.get("env") or {})
captured_timeout = kwargs.get("timeout_seconds")
return runner._BoundedHarborProcessResult(returncode=0, output_tail="", output_exceeded=False)
monkeypatch.setattr(runner, "_run_bounded_harbor_process", mock_run)
monkeypatch.setattr(runner, "_validate_harbor_job_result", lambda *_args, **_kwargs: (True, "success"))
monkeypatch.setattr("skillevaluator.provider_config._get_google_access_token", lambda **_kw: "ambient-host-adc")
explicit_env = {
"OPENAI_API_KEY": "explicit-operator-key",
"OPENAI_BASE_URL": base_url,
"PATH": "/usr/bin",
}
ok, _detail = runner._run_harbor(
dataset=tmp_path / "dataset",
agent="opencode",
job_name="explicit-job",
env_mode="docker",
model="google/gemini-3.8-flash",
jobs_dir=tmp_path / "jobs",
run_env=explicit_env,
n_attempts=1,
n_concurrent=1,
timeout_multiplier=1.0,
override_cpus=None,
override_memory_mb=None,
override_storage_mb=None,
)
assert ok is True
assert captured_env.get("OPENAI_API_KEY") == "explicit-operator-key"
assert captured_timeout is None
@pytest.mark.parametrize(
"verifier_env",
[
{"OPENAI_API_KEY": "raw-secret-token"},
{"ANTHROPIC_API_KEY": "sk-ant-raw-secret"},
{"SKILL_EVAL_LLM_CREDENTIAL_SOURCE": "ADC"},
],
)
def test_build_harbor_run_command_rejects_raw_verifier_env_secrets(
tmp_path: Path,
verifier_env: dict[str, str],
) -> None:
"""Reject raw secrets or operator-owned values in verifier_env without ${VAR} placeholder indirection."""
with pytest.raises(ValueError, match="Sensitive key detected in verifier_env"):
runner.build_harbor_run_command(
dataset_path=tmp_path / "dataset",
agent="opencode",
job_name="bad-verifier-env",
env_mode="docker",
verifier_env=verifier_env,
)
def test_run_harbor_enforces_adc_job_timeout_and_stages_credential_source(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
"""Enforce bounded ADC timeout on non-GKE jobs and stage SKILL_EVAL_LLM_CREDENTIAL_SOURCE for the verifier."""
base_url = "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi"
adc_provider = ProviderConfig(
provider="openai-compatible",
model="google/gemini-3.8-flash",
api_key="initial-adc-token",
base_url=base_url,
litellm_model="openai/google/gemini-3.8-flash",
credential_env="ADC",
base_url_env="SKILL_EVAL_LLM_BASE_URL",
)
monkeypatch.setattr("skillevaluator.provider_config._get_google_access_token", lambda **_kw: "fresh-adc-token")
monkeypatch.setattr(runner, "_get_google_access_token", lambda **_kw: "fresh-adc-token")
provider_env = runner._provider_environment(adc_provider)
assert provider_env.get("SKILL_EVAL_LLM_CREDENTIAL_SOURCE") == "ADC"
assert "SKILL_EVAL_LLM_CREDENTIAL_SOURCE" in _verifier_env_vars(provider_env)
def timeout_run(command, *args, **kwargs):
raise runner._HarborRunTimeoutError(f"timed out after {kwargs.get('timeout_seconds', 3300)}s")
monkeypatch.setattr(runner, "_run_bounded_harbor_process", timeout_run)
adc_env = {
"OPENAI_API_KEY": "initial-adc-token",
"OPENAI_BASE_URL": base_url,
"SKILL_EVAL_LLM_CREDENTIAL_SOURCE": "ADC",
"PATH": "/usr/bin",
}
ok_timeout, detail_timeout = runner._run_harbor(
dataset=tmp_path / "dataset",
agent="opencode",
job_name="timeout-job",
env_mode="docker",
model="google/gemini-3.8-flash",
jobs_dir=tmp_path / "jobs",
run_env=adc_env,
n_attempts=1,
n_concurrent=1,
timeout_multiplier=1.0,
override_cpus=None,
override_memory_mb=None,
override_storage_mb=None,
)
assert ok_timeout is False
assert "ADC token lifetime" in detail_timeout
def test_run_harbor_caps_configured_adc_timeout_and_rejects_near_expiry(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
"""Cap SKILLEVALUATOR_VERTEX_ADC_JOB_TIMEOUT_SEC to remaining token ceiling and reject <= 30s tokens."""
import time as time_mod
base_url = "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi"
fixed_now = 1_700_000_000.0
monkeypatch.setattr(time_mod, "time", lambda: fixed_now)
monkeypatch.setattr("skillevaluator.provider_config._get_google_access_token", lambda **_kw: None)
monkeypatch.setattr(runner, "_get_google_access_token", lambda **_kw: None)
captured_timeouts: list[float | None] = []
def fake_run(command, *args, **kwargs):
captured_timeouts.append(kwargs.get("timeout_seconds"))
return runner._BoundedHarborProcessResult(returncode=0, output_tail="", output_exceeded=False)
monkeypatch.setattr(runner, "_run_bounded_harbor_process", fake_run)
monkeypatch.setattr(runner, "_validate_harbor_job_result", lambda *_args, **_kwargs: (True, "success"))
# 1. 600s token + SKILLEVALUATOR_VERTEX_ADC_JOB_TIMEOUT_SEC=5000 -> capped at 540s
monkeypatch.setenv(runner.VERTEX_ADC_JOB_TIMEOUT_ENV, "5000")
adc_env_600 = {
"OPENAI_API_KEY": "short-lived-adc-token",
"OPENAI_BASE_URL": base_url,
"SKILL_EVAL_LLM_CREDENTIAL_SOURCE": "ADC",
"SKILL_EVAL_LLM_CREDENTIAL_EXPIRY": str(fixed_now + 600.0),
"PATH": "/usr/bin",
}
ok_capped, _ = runner._run_harbor(
dataset=tmp_path / "dataset",
agent="opencode",
job_name="capped-job",
env_mode="docker",
model="google/gemini-3.8-flash",
jobs_dir=tmp_path / "jobs",
run_env=adc_env_600,
n_attempts=1,
n_concurrent=1,
timeout_multiplier=1.0,
override_cpus=None,
override_memory_mb=None,
override_storage_mb=None,
)
assert ok_capped is True
assert captured_timeouts[-1] == pytest.approx(540.0)
# 2. 600s token + SKILLEVALUATOR_VERTEX_ADC_JOB_TIMEOUT_SEC=300 -> honors 300s
monkeypatch.setenv(runner.VERTEX_ADC_JOB_TIMEOUT_ENV, "300")
ok_smaller, _ = runner._run_harbor(
dataset=tmp_path / "dataset",
agent="opencode",
job_name="smaller-job",
env_mode="docker",
model="google/gemini-3.8-flash",
jobs_dir=tmp_path / "jobs",
run_env=adc_env_600,
n_attempts=1,
n_concurrent=1,
timeout_multiplier=1.0,
override_cpus=None,
override_memory_mb=None,
override_storage_mb=None,
)
assert ok_smaller is True
assert captured_timeouts[-1] == pytest.approx(300.0)
# 3. Malformed or non-positive SKILLEVALUATOR_VERTEX_ADC_JOB_TIMEOUT_SEC falls back to 540s ceiling
for bad_override in ("invalid", "-10", "nan"):
monkeypatch.setenv(runner.VERTEX_ADC_JOB_TIMEOUT_ENV, bad_override)
ok_bad_override, _ = runner._run_harbor(
dataset=tmp_path / "dataset",
agent="opencode",
job_name="bad-override-job",
env_mode="docker",
model="google/gemini-3.8-flash",
jobs_dir=tmp_path / "jobs",
run_env=adc_env_600,
n_attempts=1,
n_concurrent=1,
timeout_multiplier=1.0,
override_cpus=None,
override_memory_mb=None,
override_storage_mb=None,
)
assert ok_bad_override is True
assert captured_timeouts[-1] == pytest.approx(540.0)
# 4. 20s remaining (<= 30s safety floor) fails closed before launching Harbor
monkeypatch.delenv(runner.VERTEX_ADC_JOB_TIMEOUT_ENV, raising=False)
adc_env_near_expiry = {
**adc_env_600,
"SKILL_EVAL_LLM_CREDENTIAL_EXPIRY": str(fixed_now + 20.0),
}
ok_near, detail_near = runner._run_harbor(
dataset=tmp_path / "dataset",
agent="opencode",
job_name="near-expiry-job",
env_mode="docker",
model="google/gemini-3.8-flash",
jobs_dir=tmp_path / "jobs",
run_env=adc_env_near_expiry,
n_attempts=1,
n_concurrent=1,
timeout_multiplier=1.0,
override_cpus=None,
override_memory_mb=None,
override_storage_mb=None,
)
assert ok_near is False
assert "only 20.0s remaining" in detail_near
assert len(captured_timeouts) == 5
def test_harbor_subprocess_environment_forwards_adc_vars_only_to_non_local_modes(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
"""Forward GOOGLE_APPLICATION_CREDENTIALS and CLOUDSDK_CONFIG to docker/gke Harbor host processes but not local."""
adc_file = tmp_path / "custom_adc.json"
adc_file.write_text("{}", encoding="utf-8")
gcloud_dir = tmp_path / "gcloud-cfg"
gcloud_dir.mkdir()
monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", str(adc_file))
monkeypatch.setenv("CLOUDSDK_CONFIG", str(gcloud_dir))
adc_provider = ProviderConfig(
provider="openai-compatible",
model="google/gemini-3.8-flash",
api_key="adc-token",
base_url="https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi",
litellm_model="openai/google/gemini-3.8-flash",
credential_env="ADC",
base_url_env="SKILL_EVAL_LLM_BASE_URL",
)
provider_env = {"SKILL_EVAL_LLM_CREDENTIAL_SOURCE": "ADC", "OPENAI_API_KEY": "adc-token"}
docker_env = runner._harbor_subprocess_environment(
env_mode="docker",
provider=adc_provider,
configured_runtime_env={},
provider_env=provider_env,
agent="opencode",
agent_model="openai/google/gemini-3.8-flash",
)
assert docker_env.get("GOOGLE_APPLICATION_CREDENTIALS") == str(adc_file)
assert docker_env.get("CLOUDSDK_CONFIG") == str(gcloud_dir)
local_env = runner._harbor_subprocess_environment(
env_mode="local",
provider=adc_provider,
configured_runtime_env={},
provider_env=provider_env,
agent="opencode",
agent_model="openai/google/gemini-3.8-flash",
)
assert "GOOGLE_APPLICATION_CREDENTIALS" not in local_env
assert "CLOUDSDK_CONFIG" not in local_env
def test_harbor_subprocess_environment_forwards_gke_probe_and_autopilot_vars(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
"""Forward SKILLEVALUATOR_GKE_METADATA_PROBE_IMAGE and SKILLEVALUATOR_GKE_AUTOPILOT in gke mode and allow --ek autopilot."""
from skillevaluator.tier3.evals_config import _GKE_INFRASTRUCTURE_KWARGS, encode_environment_kwarg
monkeypatch.setenv("SKILLEVALUATOR_GKE_METADATA_PROBE_IMAGE", "us-docker.pkg.dev/my-proj/repo/python:3.12-alpine")
monkeypatch.setenv("SKILLEVALUATOR_GKE_AUTOPILOT", "true")
provider = _provider("openai")
provider_env = runner._provider_environment(provider)
gke_env = runner._harbor_subprocess_environment(
env_mode="gke",
provider=provider,
configured_runtime_env={},
provider_env=provider_env,
agent="opencode",
agent_model="openai/test-model",
)
assert gke_env.get("SKILLEVALUATOR_GKE_METADATA_PROBE_IMAGE") == "us-docker.pkg.dev/my-proj/repo/python:3.12-alpine"
assert gke_env.get("SKILLEVALUATOR_GKE_AUTOPILOT") == "true"
docker_env = runner._harbor_subprocess_environment(
env_mode="docker",
provider=provider,
configured_runtime_env={},
provider_env=provider_env,
agent="opencode",
agent_model="openai/test-model",
)
assert "SKILLEVALUATOR_GKE_METADATA_PROBE_IMAGE" not in docker_env
assert "SKILLEVALUATOR_GKE_AUTOPILOT" not in docker_env
assert "autopilot" in _GKE_INFRASTRUCTURE_KWARGS
cmd = runner.build_harbor_run_command(
dataset_path=tmp_path / "dataset",
agent="opencode",
job_name="gke-job",
env_mode="gke",
environment_kwargs={"autopilot": "true", "cluster_name": "c"},
)
assert encode_environment_kwarg("autopilot", "true") in cmd