Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

97.5 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Security regressions for the environment passed to the Harbor subprocess."""

from __future__ import annotations

import json
import os
import shutil
import subprocess
import threading
import tomllib
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path

import pytest
from harbor.utils.env import resolve_env_vars

from skillevaluator.provider_config import ProviderConfig, ProviderConfigurationError, resolve_llm_provider
from skillevaluator.tier3.harbor import runner, runtime_preflight
from skillevaluator.tier3.harbor.adapter import _verifier_env_vars
from skillevaluator.tier3.harbor.metrics import DEFAULT_METRICS, LEGACY_SCORE_POLICY
from skillevaluator.tier3_environments import HARBOR_NATIVE_ENV_MODES


def _provider(provider: str = "openai") -> ProviderConfig:
    return ProviderConfig(
        provider=provider,
        model="test-model",
        api_key="provider-key",
        base_url="https://provider.example/v1",
        litellm_model="openai/test-model",
        region="us-west-2" if provider == "bedrock" else None,
    )


def test_docker_opencode_nv_build_uses_operator_key_without_skill_runtime_env(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    provider = _provider("nv_build")
    monkeypatch.setattr(runner.os, "environ", {"PATH": "/usr/bin", "NVIDIA_API_KEY": "nvapi-test"})

    plans = runner._resolve_agent_runtime_plan(
        provider=provider,
        agents=["opencode"],
        models={"opencode": "nvidia/meta/llama-3.1-8b-instruct"},
        configured_runtime_env={},
        env_mode="docker",
    )

    plan = plans["opencode"]
    assert plan.staged_env == {"NVIDIA_API_KEY": "${NVIDIA_API_KEY}"}
    assert plan.subprocess_env["NVIDIA_API_KEY"] == "provider-key"
    assert "OPENAI_API_KEY" not in plan.subprocess_env


def test_harbor_environment_keeps_windows_program_dirs_and_python_utf8_mode(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    # Docker Desktop finds its CLI plugins under %ProgramFiles%, and Harbor needs the
    # UTF-8 mode variables on non-UTF-8 Windows locales.
    windows_env = {
        "PROGRAMDATA": r"C:\ProgramData",
        "PROGRAMFILES": r"C:\Program Files",
        "PROGRAMW6432": r"C:\Program Files",
        "PYTHONIOENCODING": "utf-8",
        "PYTHONUTF8": "1",
    }
    monkeypatch.setattr(runner.os, "environ", {"PATH": r"C:\Windows", "UNRELATED_TOKEN": "secret", **windows_env})

    env = runner._harbor_subprocess_environment(
        env_mode="docker",
        provider=_provider("nv_build"),
        configured_runtime_env={},
        provider_env={},
    )

    assert {name: env.get(name) for name in windows_env} == windows_env
    assert "UNRELATED_TOKEN" not in env


@pytest.mark.parametrize(
    ("provider_name", "runtime_model", "catalog_model", "litellm_prefix"),
    [
        ("nv_build", "nvidia/meta/llama-3.1-8b-instruct", "meta/llama-3.1-8b-instruct", "openai"),
        ("openai", "openai/gpt-4.1-mini", "gpt-4.1-mini", "openai"),
        ("openai-compatible", "openai/custom-model", "custom-model", "openai"),
        ("anthropic", "anthropic/claude-sonnet-test", "claude-sonnet-test", "anthropic"),
    ],
)
def test_opencode_runtime_plan_separates_runtime_and_catalog_model_names(
    provider_name: str,
    runtime_model: str,
    catalog_model: str,
    litellm_prefix: str,
) -> None:
    provider = ProviderConfig(
        provider=provider_name,
        model=catalog_model,
        api_key="provider-key",
        base_url="https://provider.example/v1",
        litellm_model=f"{litellm_prefix}/{catalog_model}",
    )

    plan = runner._resolve_agent_runtime_plan(
        provider=provider,
        agents=["opencode"],
        models={"opencode": runtime_model},
        configured_runtime_env={},
        env_mode="docker",
        model_sources={"opencode": "CLI"},
    )["opencode"]

    assert plan.model == runtime_model
    assert plan.provider.model == catalog_model
    assert plan.provider.litellm_model == f"{litellm_prefix}/{catalog_model}"


@pytest.mark.parametrize(
    ("provider_name", "agent", "model", "litellm_prefix"),
    [
        ("openai-compatible", "codex", "openai/gpt-oss", "openai"),
        ("anthropic", "claude-code", "anthropic/claude-custom", "anthropic"),
    ],
)
def test_non_opencode_runtime_plan_preserves_slash_prefixed_raw_model_ids(
    provider_name: str,
    agent: str,
    model: str,
    litellm_prefix: str,
) -> None:
    provider = ProviderConfig(
        provider=provider_name,
        model=model,
        api_key="provider-key",
        base_url="https://provider.example/v1",
        litellm_model=f"{litellm_prefix}/{model}",
    )

    plan = runner._resolve_agent_runtime_plan(
        provider=provider,
        agents=[agent],
        models={agent: model},
        configured_runtime_env={},
        env_mode="docker",
    )[agent]

    assert plan.model == model
    assert plan.provider.model == model
    assert plan.provider.litellm_model == f"{litellm_prefix}/{model}"


@pytest.mark.parametrize("agent", ["codex", "claude-code"])
def test_nvidia_build_docker_bridge_plan_keeps_provider_key_out_of_task_env(agent: str) -> None:
    plan = runner._resolve_agent_runtime_plan(
        provider=_provider("nv_build"),
        agents=[agent],
        models={agent: "openai/gpt-oss-120b"},
        configured_runtime_env={},
        env_mode="docker",
    )[agent]

    assert plan.staged_env == {}
    assert plan.subprocess_env["NVIDIA_API_KEY"] == "provider-key"
    assert plan.provider.provider == "nv_build"
    assert plan.provider.model == "openai/gpt-oss-120b"


@pytest.mark.parametrize(
    "owned_name",
    [
        "NVIDIA_API_KEY",
        "OPENAI_API_KEY",
        "OPENAI_BASE_URL",
        "ANTHROPIC_API_KEY",
        "ANTHROPIC_BASE_URL",
    ],
)
def test_skill_config_cannot_override_operator_owned_agent_credentials(owned_name: str) -> None:
    with pytest.raises(ValueError, match=rf"operator-owned.*{owned_name}|{owned_name}.*operator-owned"):
        runner._resolve_agent_runtime_plan(
            provider=_provider("nv_build"),
            agents=["opencode"],
            models={"opencode": "nvidia/model"},
            configured_runtime_env={owned_name: "attacker-value"},
            env_mode="docker",
        )


@pytest.mark.parametrize(
    "source_name",
    [
        "NVIDIA_API_KEY",
        "OPENAI_API_KEY",
        "OPENAI_BASE_URL",
        "ANTHROPIC_API_KEY",
        "ANTHROPIC_BASE_URL",
        "LLM_JUDGE_MODEL",
        "SKILL_EVAL_JUDGE_MODEL",
        "AWS_SECRET_ACCESS_KEY",
        "E2B_API_KEY",
        "DOCKER_HOST",
    ],
)
def test_skill_config_cannot_alias_operator_owned_credentials(
    monkeypatch: pytest.MonkeyPatch,
    source_name: str,
) -> None:
    monkeypatch.setenv(source_name, "operator-secret")

    resolved, errors = runner._resolve_runtime_env(
        {"INNOCENT_NAME": f"${{{source_name}}}"},
        env_mode="docker",
    )

    assert resolved == {}
    assert errors and "operator-owned" in errors[0]
    assert source_name in errors[0]


@pytest.mark.parametrize(
    "runtime_env",
    [
        {"SAFE_ALIAS": "${PROJECT_DEPLOY_TOKEN}"},
        {"PROJECT_DEPLOY_TOKEN": "${PROJECT_DEPLOY_TOKEN}"},
        {"safe_alias": "${ProjectDeployToken}"},
    ],
)
def test_skill_config_cannot_stage_conventional_unlisted_secret_names(
    monkeypatch: pytest.MonkeyPatch,
    runtime_env: dict[str, str],
) -> None:
    for value in runtime_env.values():
        source_name = value.removeprefix("${").removesuffix("}")
        monkeypatch.setenv(source_name, "synthetic-secret")

    resolved, errors = runner._resolve_runtime_env(runtime_env, env_mode="docker")

    assert resolved == {}
    assert errors
    assert "operator-owned" in errors[0] or "host process" in errors[0]
    assert "synthetic-secret" not in " ".join(errors)


@pytest.mark.parametrize("source_name", ["LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL"])
def test_skill_config_cannot_alias_judge_model_with_default_expansion(
    monkeypatch: pytest.MonkeyPatch,
    source_name: str,
) -> None:
    monkeypatch.setenv(source_name, "operator-model")

    resolved, errors = runner._resolve_runtime_env(
        {"INNOCENT_NAME": f"${{{source_name}:-skill-fallback}}"},
        env_mode="docker",
    )

    assert resolved == {}
    assert errors and "operator-owned" in errors[0]
    assert source_name in errors[0]


@pytest.mark.parametrize(
    "name",
    [
        "LLM_JUDGE_MODEL",
        "SKILL_EVAL_JUDGE_MODEL",
        "SKILL_EVAL_LLM_BASE_URL",
        "SKILL_EVAL_LLM_PROVIDER",
        "SKILL_EVAL_LLM_API_KEY",
    ],
)
def test_skill_config_cannot_control_evaluator_or_judge_routing(name: str) -> None:
    resolved, errors = runner._resolve_runtime_env(
        {name: "https://attacker.example/v1"},
        env_mode="docker",
    )

    assert resolved == {}
    assert errors and "host process" in errors[0]


@pytest.mark.parametrize(
    "name",
    [
        "LLM_JUDGE_MODEL",
        "SKILL_EVAL_JUDGE_MODEL",
        "LLM_JUDGE_FALLBACK_MODELS",
        "SKILL_EVAL_JUDGE_CHARACTER_FACTS",
        "LLM_JUDGE_FALLBACK_POLICY",
        "LLM_JUDGE_RETRY_POLICY",
        "SKILL_EVAL_LLM_REFUSAL_FALLBACK_MODEL",
    ],
)
@pytest.mark.parametrize("provider_name", ["openai", "openai-compatible", "anthropic", "bedrock", "nv_build"])
@pytest.mark.parametrize(
    ("configured", "expected"),
    [
        ("judge-model", "judge-model"),
        ("  judge-model  ", "judge-model"),
        ("", None),
        ("   ", None),
    ],
)
def test_provider_environment_forwards_each_host_judge_model_override(
    monkeypatch: pytest.MonkeyPatch,
    name: str,
    provider_name: str,
    configured: str,
    expected: str | None,
) -> None:
    monkeypatch.setattr(runner.os, "environ", {name: configured})

    environment = runner._provider_environment(_provider(provider_name))

    assert environment.get(name) == expected


@pytest.mark.parametrize(
    "name",
    [
        "SKILL_EVAL_LLM_MAX_RETRIES",
        "SKILL_EVAL_LLM_RETRY_BASE_DELAY",
        "SKILL_EVAL_LLM_RETRY_MAX_DELAY",
    ],
)
@pytest.mark.parametrize("provider_name", ["openai", "openai-compatible", "anthropic", "bedrock", "nv_build"])
@pytest.mark.parametrize(
    ("configured", "expected"),
    [
        ("0", "0"),
        ("  0.25  ", "0.25"),
        ("", None),
        ("   ", None),
    ],
)
def test_provider_environment_forwards_each_host_retry_override(
    monkeypatch: pytest.MonkeyPatch,
    name: str,
    provider_name: str,
    configured: str,
    expected: str | None,
) -> None:
    """Verify _provider_environment forwards stripped host retry settings and omits blank values."""
    monkeypatch.setattr(runner.os, "environ", {name: configured})

    environment = runner._provider_environment(_provider(provider_name))

    assert environment.get(name) == expected


def test_resolve_agent_runtime_plan_retains_retry_settings_in_subprocess_env(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Verify _resolve_agent_runtime_plan forwards host retry settings to Harbor subprocess_env."""
    monkeypatch.setattr(
        runner.os,
        "environ",
        {
            "PATH": "/usr/bin",
            "SKILL_EVAL_LLM_MAX_RETRIES": "0",
            "SKILL_EVAL_LLM_RETRY_BASE_DELAY": "0.1",
            "SKILL_EVAL_LLM_RETRY_MAX_DELAY": "0.5",
        },
    )

    plans = runner._resolve_agent_runtime_plan(
        provider=_provider("openai"),
        agents=["codex"],
        models={"codex": "gpt-4o-mini"},
        configured_runtime_env={},
        env_mode="docker",
    )

    subprocess_env = plans["codex"].subprocess_env
    assert subprocess_env["SKILL_EVAL_LLM_MAX_RETRIES"] == "0"
    assert subprocess_env["SKILL_EVAL_LLM_RETRY_BASE_DELAY"] == "0.1"
    assert subprocess_env["SKILL_EVAL_LLM_RETRY_MAX_DELAY"] == "0.5"
    assert "SKILL_EVAL_LLM_MAX_RETRIES" not in plans["codex"].staged_env


@pytest.mark.parametrize("source_name", ["LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL"])
def test_runtime_env_rejects_operator_owned_windows_style_variable_references(
    monkeypatch: pytest.MonkeyPatch,
    source_name: str,
) -> None:
    monkeypatch.setenv(source_name, "host-value")

    resolved, errors = runner._resolve_runtime_env(
        {"SAFE_ALIAS": f"%{source_name}%"},
        env_mode="docker",
    )

    assert resolved == {}
    assert errors and source_name in errors[0]
    assert "operator-owned" in errors[0]


def test_runtime_env_preserves_platform_expansion_for_non_owned_percent_references(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    expanded: list[str] = []

    def expandvars(value: str) -> str:
        expanded.append(value)
        return value.replace("%HOST_AGENT_LABEL%", "host-value")

    monkeypatch.setattr(runner.os.path, "expandvars", expandvars)

    resolved, errors = runner._resolve_runtime_env(
        {"SAFE_ALIAS": "%HOST_AGENT_LABEL%"},
        env_mode="docker",
    )

    assert errors == []
    assert resolved == {"SAFE_ALIAS": "host-value"}
    assert expanded == ["%HOST_AGENT_LABEL%"]


def test_runtime_env_allows_mode_unrelated_backend_name() -> None:
    resolved, errors = runner._resolve_runtime_env(
        {"API_HOST": "ordinary-task-value"},
        env_mode="docker",
    )

    assert errors == []
    assert resolved == {"API_HOST": "ordinary-task-value"}


def test_runtime_env_rejects_selected_backend_control_name() -> None:
    resolved, errors = runner._resolve_runtime_env(
        {"API_HOST": "attacker-controlled"},
        env_mode="beam",
    )

    assert resolved == {}
    assert errors == ["harbor.runtime_env.API_HOST controls the host process and is not allowed"]


def test_custom_only_does_not_force_the_standard_judge_model_into_custom_verifiers() -> None:
    provider_env = {
        "LLM_JUDGE_MODEL": "legacy-judge-model",
        "SKILL_EVAL_JUDGE_MODEL": "judge-model",
    }

    assert runner._job_judge_verifier_env(provider_env, "custom_only") == {}
    assert runner._job_judge_subprocess_env(provider_env, "custom_only") == {}
    assert runner._judge_model_config(_provider("nv_build"), provider_env, "custom_only") == {
        "enabled": False,
    }


@pytest.mark.parametrize(
    ("provider_env", "expected"),
    [
        (
            {"SKILL_EVAL_JUDGE_MODEL": "operator-model"},
            {
                "LLM_JUDGE_MODEL": "${SKILL_EVAL_JUDGE_MODEL}",
                "SKILL_EVAL_JUDGE_MODEL": "${SKILL_EVAL_JUDGE_MODEL}",
            },
        ),
        (
            {"LLM_JUDGE_MODEL": "legacy-model", "SKILL_EVAL_JUDGE_MODEL": "operator-model"},
            {
                "LLM_JUDGE_MODEL": "${LLM_JUDGE_MODEL}",
                "SKILL_EVAL_JUDGE_MODEL": "${LLM_JUDGE_MODEL}",
            },
        ),
    ],
)
def test_standard_judge_override_occupies_the_highest_precedence_verifier_key(
    provider_env: dict[str, str],
    expected: dict[str, str],
) -> None:
    assert runner._job_judge_verifier_env(provider_env, "default") == expected


@pytest.mark.parametrize("grading_mode", ["default", "default_plus_custom"])
def test_standard_judge_fallback_is_forwarded_only_through_the_verifier_job(
    grading_mode: str,
) -> None:
    provider_env = {"LLM_JUDGE_FALLBACK_MODELS": "fallback-one,fallback-two"}

    assert runner._job_judge_verifier_env(provider_env, grading_mode) == {
        "LLM_JUDGE_FALLBACK_MODELS": "${LLM_JUDGE_FALLBACK_MODELS}",
    }
    assert runner._job_judge_subprocess_env(provider_env, grading_mode) == {
        "LLM_JUDGE_FALLBACK_MODELS": "fallback-one,fallback-two",
    }
    assert runner._job_judge_verifier_env(provider_env, "custom_only") == {}
    assert runner._job_judge_subprocess_env(provider_env, "custom_only") == {}


@pytest.mark.parametrize(
    "host_env",
    [
        {"LLM_JUDGE_MODEL": "host-legacy"},
        {"SKILL_EVAL_JUDGE_MODEL": "host-canonical"},
        {"LLM_JUDGE_MODEL": "host-legacy", "SKILL_EVAL_JUDGE_MODEL": "host-canonical"},
    ],
)
def test_standard_judge_override_replaces_both_native_aliases_before_harbor_resolution(
    monkeypatch: pytest.MonkeyPatch,
    host_env: dict[str, str],
) -> None:
    for name in ("LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL"):
        monkeypatch.delenv(name, raising=False)
    for name, value in host_env.items():
        monkeypatch.setenv(name, value)
    provider_env = runner._provider_environment(_provider())
    task_env = {
        "LLM_JUDGE_MODEL": "task-legacy",
        "SKILL_EVAL_JUDGE_MODEL": "${SKILL_EVAL_JUDGE_MODEL}",
    }

    merged = {**task_env, **runner._job_judge_verifier_env(provider_env, "default")}
    resolved = resolve_env_vars(merged)
    selected = host_env.get("LLM_JUDGE_MODEL") or host_env["SKILL_EVAL_JUDGE_MODEL"]

    assert resolved == {
        "LLM_JUDGE_MODEL": selected,
        "SKILL_EVAL_JUDGE_MODEL": selected,
    }


@pytest.mark.parametrize("grading_mode", ["default", "default_plus_custom"])
@pytest.mark.parametrize(
    "host_env",
    [
        {"LLM_JUDGE_MODEL": "host-legacy"},
        {"SKILL_EVAL_JUDGE_MODEL": "host-canonical"},
        {"LLM_JUDGE_MODEL": "host-legacy", "SKILL_EVAL_JUDGE_MODEL": "host-canonical"},
    ],
)
def test_standard_judge_override_resolves_both_aliases_for_separate_verifier_startup(
    monkeypatch: pytest.MonkeyPatch,
    grading_mode: str,
    host_env: dict[str, str],
) -> None:
    for name in ("LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL"):
        monkeypatch.delenv(name, raising=False)
    for name, value in host_env.items():
        monkeypatch.setenv(name, value)
    provider_env = runner._provider_environment(_provider())
    subprocess_aliases = runner._job_judge_subprocess_env(provider_env, grading_mode)

    with monkeypatch.context() as resolver_env:
        for name in ("LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL"):
            resolver_env.delenv(name, raising=False)
        for name, value in subprocess_aliases.items():
            resolver_env.setenv(name, value)
        resolved = resolve_env_vars(
            {
                "LLM_JUDGE_MODEL": "${LLM_JUDGE_MODEL}",
                "SKILL_EVAL_JUDGE_MODEL": "${SKILL_EVAL_JUDGE_MODEL}",
            }
        )

    selected = host_env.get("LLM_JUDGE_MODEL") or host_env["SKILL_EVAL_JUDGE_MODEL"]
    assert resolved == {
        "LLM_JUDGE_MODEL": selected,
        "SKILL_EVAL_JUDGE_MODEL": selected,
    }


@pytest.mark.parametrize(
    ("configured_model", "expected_source"),
    [(None, "provider default"), ("operator-model", "SKILL_EVAL_LLM_MODEL")],
)
def test_judge_model_config_distinguishes_provider_default_from_explicit_model(
    monkeypatch: pytest.MonkeyPatch,
    configured_model: str | None,
    expected_source: str,
) -> None:
    if configured_model is None:
        monkeypatch.delenv("SKILL_EVAL_LLM_MODEL", raising=False)
    else:
        monkeypatch.setenv("SKILL_EVAL_LLM_MODEL", configured_model)
    provider = _provider("openai")

    config = runner._judge_model_config(provider, runner._provider_environment(provider), "default")

    assert config == {
        "enabled": True,
        "provider": "openai",
        "model": "test-model",
        "source": expected_source,
        "override_applied": False,
        "reasoning_effort": None,
    }


@pytest.mark.parametrize(
    ("provider_name", "judge_model", "effort"),
    [
        ("openai", "gpt-6.1-sol", "medium"),
        ("anthropic", "claude-opus-5-5", "medium"),
        ("openai", "gpt-5.6-sol", None),
        ("bedrock", "us.anthropic.claude-opus-5-5", None),
    ],
)
def test_judge_model_config_records_the_judge_reasoning_effort(
    provider_name: str, judge_model: str, effort: str | None
) -> None:
    provider = _provider(provider_name)

    config = runner._judge_model_config(provider, {"LLM_JUDGE_MODEL": judge_model}, "default")

    assert config["reasoning_effort"] == effort


def test_mixed_agents_receive_disjoint_credentials(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(
        runner.os,
        "environ",
        {
            "PATH": "/usr/bin",
            "NVIDIA_API_KEY": "nvapi-test",
            "ANTHROPIC_API_KEY": "anthropic-test",
            "OPENAI_API_KEY": "openai-test",
            "OPENAI_BASE_URL": "https://api.openai.com/v1",
        },
    )

    plans = runner._resolve_agent_runtime_plan(
        provider=_provider("nv_build"),
        agents=["opencode", "claude-code", "codex"],
        models={
            "opencode": "nvidia/nvidia/nemotron-3-nano-30b-a3b",
            "claude-code": "nvidia/nemotron-3.5-lightning-30b-a3b",
            "codex": "nvidia/nemotron-3-nano-30b-a3b",
        },
        configured_runtime_env={},
        env_mode="docker",
        model_sources={"opencode": "provider", "claude-code": "CLI", "codex": "CLI"},
    )

    assert plans["opencode"].staged_env == {"NVIDIA_API_KEY": "${NVIDIA_API_KEY}"}
    assert plans["claude-code"].staged_env == {}
    assert plans["codex"].staged_env == {}
    assert plans["opencode"].provider.provider == "nv_build"
    assert plans["claude-code"].provider.provider == "nv_build"
    assert plans["codex"].provider.provider == "nv_build"
    assert "ANTHROPIC_API_KEY" not in plans["opencode"].subprocess_env
    assert "OPENAI_API_KEY" not in plans["opencode"].subprocess_env
    assert "NVIDIA_API_KEY" in plans["claude-code"].subprocess_env  # evaluator verifier only
    assert "OPENAI_API_KEY" not in plans["claude-code"].subprocess_env
    assert "ANTHROPIC_API_KEY" not in plans["codex"].subprocess_env
    with pytest.raises(TypeError):
        plans["opencode"].staged_env["MUTATE"] = "forbidden"  # type: ignore[index]


@pytest.mark.parametrize(
    ("provider_name", "agent"),
    [
        ("bedrock", "codex"),
        ("bedrock", "opencode"),
    ],
)
def test_incompatible_provider_agent_pairs_fail_before_harbor(provider_name: str, agent: str) -> None:
    with pytest.raises(ValueError, match="does not support"):
        runner._resolve_agent_runtime_plan(
            provider=_provider(provider_name),
            agents=[agent],
            models={agent: "test-model"},
            configured_runtime_env={},
            env_mode="docker",
            model_sources={agent: "CLI"},
        )


def test_openai_mixed_agents_receive_isolated_native_credentials(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(
        runner.os,
        "environ",
        {
            "PATH": "/usr/bin",
            "ANTHROPIC_API_KEY": "anthropic-agent-key",
        },
    )

    plans = runner._resolve_agent_runtime_plan(
        provider=_provider("openai"),
        agents=["codex", "claude-code"],
        models={"codex": "test-model", "claude-code": "anthropic/claude-sonnet-4-5"},
        configured_runtime_env={},
        env_mode="docker",
        model_sources={"codex": "public provider default", "claude-code": "CLI"},
    )

    assert plans["codex"].staged_env == {
        "OPENAI_API_KEY": "${OPENAI_API_KEY}",
        "OPENAI_BASE_URL": "${OPENAI_BASE_URL}",
    }
    assert plans["claude-code"].staged_env == {"ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}"}
    assert plans["codex"].provider.provider == "openai"
    assert plans["claude-code"].provider.provider == "anthropic"
    assert plans["claude-code"].provider.model == "claude-sonnet-4-5"
    assert "ANTHROPIC_API_KEY" not in plans["codex"].subprocess_env
    assert plans["claude-code"].subprocess_env["ANTHROPIC_API_KEY"] == "anthropic-agent-key"
    assert plans["claude-code"].subprocess_env["OPENAI_API_KEY"] == "provider-key"


@pytest.mark.parametrize(
    ("provider_name", "env_mode", "configured_base_url", "expected_base_url"),
    [
        ("openai", "docker", "https://agent-gateway.example", "https://agent-gateway.example"),
        ("openai", "e2b", "https://agent-gateway.example/v1", "https://agent-gateway.example"),
        (
            "openai-compatible",
            "docker",
            "https://agent-gateway.example/team/v1/",
            "https://agent-gateway.example/team",
        ),
        ("nv_build", "e2b", "https://agent-gateway.example", "https://agent-gateway.example"),
        ("nv_build", "e2b", "https://agent-gateway.example/v1", "https://agent-gateway.example"),
        (
            "nv_build",
            "e2b",
            "https://agent-gateway.example/team/v1/",
            "https://agent-gateway.example/team",
        ),
    ],
)
def test_independent_claude_base_url_is_canonical_across_the_runtime_plan(
    monkeypatch: pytest.MonkeyPatch,
    provider_name: str,
    env_mode: str,
    configured_base_url: str,
    expected_base_url: str,
) -> None:
    monkeypatch.setattr(
        runner.os,
        "environ",
        {
            "PATH": "/usr/bin",
            "ANTHROPIC_API_KEY": "anthropic-agent-key",
            "ANTHROPIC_BASE_URL": configured_base_url,
        },
    )
    evaluator_provider = _provider(provider_name)

    plan = runner._resolve_agent_runtime_plan(
        provider=evaluator_provider,
        agents=["claude-code"],
        models={"claude-code": "anthropic/claude-sonnet-4-5"},
        configured_runtime_env={},
        env_mode=env_mode,
        model_sources={"claude-code": "CLI"},
    )["claude-code"]

    assert plan.staged_env == {
        "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}",
        "ANTHROPIC_BASE_URL": "${ANTHROPIC_BASE_URL}",
    }
    assert plan.subprocess_env["ANTHROPIC_API_KEY"] == "anthropic-agent-key"
    assert plan.subprocess_env["ANTHROPIC_BASE_URL"] == expected_base_url
    assert plan.provider.provider == "anthropic"
    assert plan.provider.api_key == "anthropic-agent-key"
    assert plan.provider.base_url == expected_base_url
    assert plan.provider.base_url + "/v1/messages" == expected_base_url + "/v1/messages"
    assert "/v1/v1/" not in plan.provider.base_url + "/v1/messages"

    assert evaluator_provider.api_key == "provider-key"
    assert evaluator_provider.base_url == "https://provider.example/v1"
    if provider_name == "nv_build":
        assert plan.subprocess_env["NVIDIA_API_KEY"] == "provider-key"
        assert "OPENAI_API_KEY" not in plan.subprocess_env
    else:
        assert plan.subprocess_env["OPENAI_API_KEY"] == "provider-key"
        assert plan.subprocess_env["OPENAI_BASE_URL"] == "https://provider.example/v1"


@pytest.mark.parametrize(
    ("provider_name", "env_mode"),
    [
        ("openai", "docker"),
        ("openai-compatible", "docker"),
        ("nv_build", "e2b"),
    ],
)
@pytest.mark.parametrize(
    "configured_base_url",
    [
        "   ",
        "agent-gateway.example/v1",
        "https://agent-gateway.example/v1/messages",
        "https://url-user:[email protected]/v1",
        "https://agent-gateway.example/v1?token=url-secret",
        "https://agent-gateway.example/v1\n",
        "https://agent-gateway.example/%2576%2531/%256dessages",
    ],
)
def test_invalid_independent_claude_base_url_fails_before_harbor_without_echoing_secrets(
    monkeypatch: pytest.MonkeyPatch,
    provider_name: str,
    env_mode: str,
    configured_base_url: str,
) -> None:
    monkeypatch.setattr(
        runner.os,
        "environ",
        {
            "PATH": "/usr/bin",
            "ANTHROPIC_API_KEY": "anthropic-agent-key",
            "ANTHROPIC_BASE_URL": configured_base_url,
        },
    )

    with pytest.raises(ProviderConfigurationError) as exc_info:
        runner._resolve_agent_runtime_plan(
            provider=_provider(provider_name),
            agents=["claude-code"],
            models={"claude-code": "anthropic/claude-sonnet-4-5"},
            configured_runtime_env={},
            env_mode=env_mode,
            model_sources={"claude-code": "CLI"},
        )

    message = str(exc_info.value)
    assert "ANTHROPIC_BASE_URL" in message
    assert configured_base_url not in message
    assert "url-user" not in message
    assert "url-secret" not in message
    assert "anthropic-agent-key" not in message


@pytest.mark.parametrize("configured_base_url", [None, ""])
def test_unset_independent_claude_base_url_remains_absent(
    monkeypatch: pytest.MonkeyPatch,
    configured_base_url: str | None,
) -> None:
    host_env = {
        "PATH": "/usr/bin",
        "ANTHROPIC_API_KEY": "anthropic-agent-key",
    }
    if configured_base_url is not None:
        host_env["ANTHROPIC_BASE_URL"] = configured_base_url
    monkeypatch.setattr(runner.os, "environ", host_env)

    plan = runner._resolve_agent_runtime_plan(
        provider=_provider("openai"),
        agents=["claude-code"],
        models={"claude-code": "anthropic/claude-sonnet-4-5"},
        configured_runtime_env={},
        env_mode="docker",
        model_sources={"claude-code": "CLI"},
    )["claude-code"]

    assert "ANTHROPIC_BASE_URL" not in plan.staged_env
    assert "ANTHROPIC_BASE_URL" not in plan.subprocess_env
    assert plan.provider.base_url is None


def test_independent_claude_base_url_optional_normalizer_result_is_not_staged(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(
        runner.os,
        "environ",
        {
            "PATH": "/usr/bin",
            "ANTHROPIC_API_KEY": "anthropic-agent-key",
            "ANTHROPIC_BASE_URL": "https://agent-gateway.example/v1",
        },
    )

    def no_configured_base_url(_value: str, *, variable: str) -> None:
        assert variable == "ANTHROPIC_BASE_URL"

    monkeypatch.setattr(runner, "_normalize_anthropic_base_url", no_configured_base_url)

    plan = runner._resolve_agent_runtime_plan(
        provider=_provider("openai"),
        agents=["claude-code"],
        models={"claude-code": "anthropic/claude-sonnet-4-5"},
        configured_runtime_env={},
        env_mode="docker",
        model_sources={"claude-code": "CLI"},
    )["claude-code"]

    assert "ANTHROPIC_BASE_URL" not in plan.staged_env
    assert "ANTHROPIC_BASE_URL" not in plan.subprocess_env
    assert plan.provider.base_url is None


@pytest.mark.live
@pytest.mark.skipif(shutil.which("claude") is None, reason="Installed Claude Code CLI is required")
def test_installed_claude_uses_canonical_independent_gateway_from_runtime_plan(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    request_paths: list[str] = []
    response_events = [
        (
            "message_start",
            {
                "type": "message_start",
                "message": {
                    "id": "msg_test",
                    "type": "message",
                    "role": "assistant",
                    "model": "claude-sonnet-4-5",
                    "content": [],
                    "stop_reason": None,
                    "stop_sequence": None,
                    "usage": {"input_tokens": 1, "output_tokens": 0},
                },
            },
        ),
        (
            "content_block_start",
            {
                "type": "content_block_start",
                "index": 0,
                "content_block": {"type": "text", "text": ""},
            },
        ),
        (
            "content_block_delta",
            {
                "type": "content_block_delta",
                "index": 0,
                "delta": {"type": "text_delta", "text": "gateway-ok"},
            },
        ),
        ("content_block_stop", {"type": "content_block_stop", "index": 0}),
        (
            "message_delta",
            {
                "type": "message_delta",
                "delta": {"stop_reason": "end_turn", "stop_sequence": None},
                "usage": {"output_tokens": 1},
            },
        ),
        ("message_stop", {"type": "message_stop"}),
    ]
    response_body = "".join(
        f"event: {event}\ndata: {json.dumps(payload, separators=(',', ':'))}\n\n" for event, payload in response_events
    ).encode()

    class RecordingHandler(BaseHTTPRequestHandler):
        def log_message(self, _format: str, *args: object) -> None:
            del args

        def do_POST(self) -> None:
            content_length = int(self.headers.get("content-length", "0"))
            self.rfile.read(content_length)
            request_paths.append(self.path.partition("?")[0])
            self.send_response(200)
            self.send_header("content-type", "text/event-stream")
            self.send_header("content-length", str(len(response_body)))
            self.end_headers()
            self.wfile.write(response_body)

    claude_path = shutil.which("claude")
    assert claude_path is not None
    with ThreadingHTTPServer(("127.0.0.1", 0), RecordingHandler) as server:
        server_thread = threading.Thread(target=server.serve_forever, daemon=True)
        server_thread.start()
        configured_base_url = f"http://127.0.0.1:{server.server_port}/team/v1"
        monkeypatch.setattr(
            runner.os,
            "environ",
            {
                "PATH": os.environ["PATH"],
                "HOME": str(tmp_path),
                "CLAUDE_CONFIG_DIR": str(tmp_path / "claude"),
                "ANTHROPIC_API_KEY": "test-agent-key",
                "ANTHROPIC_BASE_URL": configured_base_url,
            },
        )
        plan = runner._resolve_agent_runtime_plan(
            provider=_provider("openai"),
            agents=["claude-code"],
            models={"claude-code": "anthropic/claude-sonnet-4-5"},
            configured_runtime_env={},
            env_mode="docker",
            model_sources={"claude-code": "CLI"},
        )["claude-code"]

        try:
            result = subprocess.run(
                [
                    claude_path,
                    "--bare",
                    "--print",
                    "--no-session-persistence",
                    "--disable-slash-commands",
                    "--model",
                    plan.provider.model,
                    "Reply only with gateway-ok",
                ],
                cwd=tmp_path,
                env=dict(plan.subprocess_env),
                capture_output=True,
                text=True,
                timeout=30,
                check=False,
            )
        finally:
            server.shutdown()
            server_thread.join(timeout=5)

    expected_root = f"http://127.0.0.1:{server.server_port}/team"
    assert plan.provider.base_url == expected_root
    assert plan.subprocess_env["ANTHROPIC_BASE_URL"] == expected_root
    assert plan.subprocess_env["ANTHROPIC_API_KEY"] == "test-agent-key"
    assert plan.subprocess_env["OPENAI_API_KEY"] == "provider-key"
    assert result.returncode == 0, result.stderr
    assert result.stdout.strip() == "gateway-ok"
    assert request_paths
    assert set(request_paths) == {"/team/v1/messages"}


def test_anthropic_mixed_agents_receive_isolated_native_credentials(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(
        runner.os,
        "environ",
        {
            "PATH": "/usr/bin",
            "OPENAI_API_KEY": "openai-agent-key",
            "OPENAI_BASE_URL": "https://api.openai.com/v1",
        },
    )

    plans = runner._resolve_agent_runtime_plan(
        provider=_provider("anthropic"),
        agents=["claude-code", "codex"],
        models={"claude-code": "test-model", "codex": "openai/gpt-5.4-mini"},
        configured_runtime_env={},
        env_mode="docker",
        model_sources={"claude-code": "public provider default", "codex": "CLI"},
    )

    assert plans["claude-code"].staged_env == {
        "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}",
        "ANTHROPIC_BASE_URL": "${ANTHROPIC_BASE_URL}",
    }
    assert plans["codex"].staged_env == {
        "OPENAI_API_KEY": "${OPENAI_API_KEY}",
        "OPENAI_BASE_URL": "${OPENAI_BASE_URL}",
    }
    assert plans["claude-code"].provider.provider == "anthropic"
    assert plans["codex"].provider.provider == "openai-compatible"
    assert plans["codex"].provider.model == "gpt-5.4-mini"
    assert "OPENAI_API_KEY" not in plans["claude-code"].subprocess_env
    assert plans["codex"].subprocess_env["OPENAI_API_KEY"] == "openai-agent-key"
    assert plans["codex"].subprocess_env["ANTHROPIC_API_KEY"] == "provider-key"


def test_anthropic_legacy_v1_base_is_canonical_for_verifier_and_native_agents(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(runner.os, "environ", {"PATH": "/usr/bin"})
    provider = resolve_llm_provider(
        {
            "SKILL_EVAL_LLM_PROVIDER": "anthropic",
            "ANTHROPIC_API_KEY": "anthropic-provider-key",
            "ANTHROPIC_BASE_URL": "https://gateway.example/team/v1/",
        }
    )

    verifier_env = runner._provider_environment(provider)
    plans = runner._resolve_agent_runtime_plan(
        provider=provider,
        agents=["claude-code", "opencode"],
        models={
            "claude-code": "claude-sonnet-4-5",
            "opencode": "anthropic/claude-sonnet-4-5",
        },
        configured_runtime_env={},
        env_mode="docker",
        model_sources={"claude-code": "public provider default", "opencode": "CLI"},
    )

    assert provider.base_url == "https://gateway.example/team"
    assert verifier_env["ANTHROPIC_BASE_URL"] == "https://gateway.example/team"
    for agent in ("claude-code", "opencode"):
        assert plans[agent].staged_env == {
            "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}",
            "ANTHROPIC_BASE_URL": "${ANTHROPIC_BASE_URL}",
        }
        assert plans[agent].subprocess_env["ANTHROPIC_BASE_URL"] == "https://gateway.example/team"
        assert plans[agent].provider.base_url == "https://gateway.example/team"
        assert plans[agent].subprocess_env["ANTHROPIC_BASE_URL"] + "/v1/messages" == (
            "https://gateway.example/team/v1/messages"
        )
        assert "/v1/v1/" not in plans[agent].subprocess_env["ANTHROPIC_BASE_URL"] + "/v1/messages"


def test_openai_compatible_codex_stages_selected_provider_pair() -> None:
    plans = runner._resolve_agent_runtime_plan(
        provider=_provider("openai-compatible"),
        agents=["codex"],
        models={"codex": "responses-model"},
        configured_runtime_env={},
        env_mode="docker",
    )

    assert plans["codex"].staged_env == {
        "OPENAI_API_KEY": "${OPENAI_API_KEY}",
        "OPENAI_BASE_URL": "${OPENAI_BASE_URL}",
    }


def test_local_anthropic_opencode_is_rejected_explicitly() -> None:
    with pytest.raises(ValueError, match="local mode"):
        runner._resolve_agent_runtime_plan(
            provider=_provider("anthropic"),
            agents=["opencode"],
            models={"opencode": "anthropic/claude-test"},
            configured_runtime_env={},
            env_mode="local",
        )


def test_local_bedrock_agent_is_rejected_explicitly(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setenv("AWS_ACCESS_KEY_ID", "access")
    monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "secret")
    with pytest.raises(ValueError, match="local mode"):
        runner._resolve_agent_runtime_plan(
            provider=_provider("bedrock"),
            agents=["claude-code"],
            models={"claude-code": "us.anthropic.model"},
            configured_runtime_env={},
            env_mode="local",
        )


def test_docker_bedrock_claude_activates_bedrock_mode(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(
        runner.os,
        "environ",
        {
            "PATH": "/usr/bin",
            "AWS_ACCESS_KEY_ID": "access",
            "AWS_SECRET_ACCESS_KEY": "secret",
            "AWS_REGION": "us-west-2",
        },
    )

    plans = runner._resolve_agent_runtime_plan(
        provider=_provider("bedrock"),
        agents=["claude-code"],
        models={"claude-code": "us.anthropic.model"},
        configured_runtime_env={},
        env_mode="docker",
    )

    plan = plans["claude-code"]
    assert plan.staged_env["CLAUDE_CODE_USE_BEDROCK"] == "${CLAUDE_CODE_USE_BEDROCK}"
    assert plan.subprocess_env["CLAUDE_CODE_USE_BEDROCK"] == "1"
    assert plan.staged_env["AWS_ACCESS_KEY_ID"] == "${AWS_ACCESS_KEY_ID}"


def test_run_harbor_eval_stages_per_agent_credential_trees(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path,
) -> None:
    skill = tmp_path / "demo"
    (skill / "evals").mkdir(parents=True)
    (skill / "evals" / "evals.json").write_text("[]\n", encoding="utf-8")
    provider = _provider("nv_build")
    emitted: list[tuple[str, bool, dict[str, str], dict[str, str]]] = []
    launched: dict[str, tuple[str, str, dict[str, str], dict[str, str]]] = {}
    launched_task_selectors: dict[str, list[str]] = {}
    collected: dict[str, object] = {}

    monkeypatch.setenv("LLM_JUDGE_MODEL", "legacy-judge-model")
    monkeypatch.setenv("SKILL_EVAL_JUDGE_MODEL", "judge-model")
    monkeypatch.setenv("LLM_JUDGE_FALLBACK_MODELS", "fallback-one,fallback-two")
    monkeypatch.setenv("SKILL_EVAL_JUDGE_CHARACTER_FACTS", "1")

    monkeypatch.setattr(runner, "resolve_llm_provider", lambda: provider)
    monkeypatch.setattr(
        runner,
        "load_evals_config",
        lambda _path: ({"harbor": {"task_source": "evals_json"}}, None),
    )
    monkeypatch.setattr(runner, "find_evals_file", lambda _path: skill / "evals" / "evals.json")
    monkeypatch.setattr(runner, "_check_prerequisites", lambda **_kwargs: [])

    def emit(_skill, target, *, with_skill, runtime_env, **_kwargs):
        task = target / "case-001"
        task.mkdir(parents=True)
        (task / "task.toml").write_text(
            'schema_version = "1.3"\n\n[task]\nname = "publisher/presentation-name"\n\n'
            '[metadata]\nentry_id = "logical-case-001"\n',
            encoding="utf-8",
        )
        emitted.append(
            (
                str(target.relative_to(tmp_path / "results")),
                with_skill,
                dict(runtime_env),
                dict(_kwargs["verifier_env"]),
            )
        )
        return [task]

    def launch(**kwargs):
        assert (kwargs["jobs_dir"].parent / "dataset_snapshot.json").is_file()
        launched_task_selectors[kwargs["agent"]] = list(kwargs["task_names"])
        launched[kwargs["agent"]] = (
            str(kwargs["with_skill"].relative_to(tmp_path / "results")),
            str(kwargs["baseline"].relative_to(tmp_path / "results")),
            dict(kwargs["run_env"]),
            dict(kwargs["verifier_env"]),
        )
        return []

    monkeypatch.setattr(runner, "generate_harbor_tasks", emit)
    monkeypatch.setattr(runner, "_run_agent_pair", launch)
    monkeypatch.setattr(
        runtime_preflight,
        "probe_model",
        lambda selected_provider: runtime_preflight.ModelProbeResult(
            True,
            selected_provider.provider,
            selected_provider.model,
            f"model {selected_provider.model} is available",
        ),
    )
    monkeypatch.setattr(
        runner,
        "collect_harbor_results",
        lambda **kwargs: (
            collected.update(kwargs)
            or {
                "execution_status": "complete",
                "execution_errors": [],
                "metrics": list(DEFAULT_METRICS),
                "score_policy": LEGACY_SCORE_POLICY,
                "attempt_policy": {
                    "score_policy": LEGACY_SCORE_POLICY,
                    "score_definition": "historical six-metric mean",
                },
                "agents": {},
            }
        ),
    )
    monkeypatch.setattr(runner, "render_agent_eval_html_report", lambda *_args, **_kwargs: tmp_path / "report.html")
    result = runner.run_harbor_eval(
        skill,
        ["opencode", "claude-code"],
        agent_models={
            "opencode": "nvidia/nvidia/nemotron-3-nano-30b-a3b",
            "claude-code": "nvidia/nemotron-3.5-lightning-30b-a3b",
        },
        output_dir=tmp_path / "results",
        env_mode="docker",
        keep_harbor_jobs=True,
        agent_runtime_preflight=False,
    )

    assert "error" not in result
    assert result["score_policy"] == result["attempt_policy"]["score_policy"] == LEGACY_SCORE_POLICY
    assert result["attempt_policy"]["score_definition"] == "historical six-metric mean"
    assert result["attempt_policy"]["max_attempts"] == 1
    persisted = json.loads(Path(result["result_path"]).read_text(encoding="utf-8"))
    assert persisted["attempt_policy"]["score_policy"] == LEGACY_SCORE_POLICY
    assert {
        (path.split("/")[-2], path.split("/")[-1], with_skill)
        for path, with_skill, _runtime_env, _verifier_env in emitted
    } == {
        ("opencode", "with", True),
        ("opencode", "without", False),
        ("claude-code", "with", True),
        ("claude-code", "without", False),
    }
    staged = {
        (path.split("/")[-2], path.split("/")[-1]): runtime_env
        for path, _with_skill, runtime_env, _verifier_env in emitted
    }
    assert staged[("opencode", "with")] == {"NVIDIA_API_KEY": "${NVIDIA_API_KEY}"}
    assert staged[("claude-code", "with")] == {}
    for _path, _with_skill, runtime_env, verifier_env in emitted:
        assert "LLM_JUDGE_MODEL" not in runtime_env
        assert "SKILL_EVAL_JUDGE_MODEL" not in runtime_env
        assert "LLM_JUDGE_FALLBACK_MODELS" not in runtime_env
        assert "SKILL_EVAL_JUDGE_CHARACTER_FACTS" not in runtime_env
        assert "LLM_JUDGE_MODEL" not in verifier_env
        assert "SKILL_EVAL_JUDGE_MODEL" not in verifier_env
        assert verifier_env["LLM_JUDGE_FALLBACK_MODELS"] == "${LLM_JUDGE_FALLBACK_MODELS}"
        assert verifier_env["SKILL_EVAL_JUDGE_CHARACTER_FACTS"] == "${SKILL_EVAL_JUDGE_CHARACTER_FACTS}"
    assert launched["opencode"][0].endswith("_harbor-tasks/opencode/with")
    assert launched["claude-code"][1].endswith("_harbor-tasks/claude-code/without")
    assert launched_task_selectors == {"opencode": ["case-001"], "claude-code": ["case-001"]}
    assert collected["expected_case_ids"] == ["logical-case-001"]
    assert collected["case_id_by_task_selector"] == {"case-001": "logical-case-001"}
    assert launched["opencode"][2]["LLM_JUDGE_MODEL"] == "legacy-judge-model"
    assert launched["opencode"][2]["SKILL_EVAL_JUDGE_MODEL"] == "legacy-judge-model"
    assert launched["opencode"][2]["LLM_JUDGE_FALLBACK_MODELS"] == "fallback-one,fallback-two"
    assert launched["opencode"][2]["SKILL_EVAL_JUDGE_CHARACTER_FACTS"] == "1"
    assert "ANTHROPIC_API_KEY" not in launched["opencode"][2]
    assert "OPENAI_API_KEY" not in launched["opencode"][2]
    assert launched["claude-code"][2]["NVIDIA_API_KEY"] == "provider-key"
    assert launched["opencode"][3] == {
        "LLM_JUDGE_FALLBACK_MODELS": "${LLM_JUDGE_FALLBACK_MODELS}",
        "SKILL_EVAL_JUDGE_CHARACTER_FACTS": "${SKILL_EVAL_JUDGE_CHARACTER_FACTS}",
        "LLM_JUDGE_MODEL": "${LLM_JUDGE_MODEL}",
        "SKILL_EVAL_JUDGE_MODEL": "${LLM_JUDGE_MODEL}",
    }
    assert launched["claude-code"][3] == launched["opencode"][3]
    assert result["run_config"]["judge"] == {
        "enabled": True,
        "provider": "nv_build",
        "model": "legacy-judge-model",
        "source": "LLM_JUDGE_MODEL",
        "override_applied": True,
        "reasoning_effort": None,
        "character_facts": "1",
        "catalog_verification": "degraded",
    }


@pytest.mark.parametrize("grading_mode", ["default", "default_plus_custom"])
@pytest.mark.parametrize(
    "judge_alias",
    ["LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL", "LLM_JUDGE_FALLBACK_MODELS", "SKILL_EVAL_JUDGE_CHARACTER_FACTS"],
)
@pytest.mark.parametrize("judge_scope", ["task", "step"])
def test_native_runtime_rejects_task_owned_judge_selection_through_real_staging(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    grading_mode: str,
    judge_alias: str,
    judge_scope: str,
) -> None:
    skill = tmp_path / "demo"
    evals = skill / "evals"
    native_task = evals / "harbor" / "case-001"
    native_task.mkdir(parents=True)
    (skill / "SKILL.md").write_text("# Demo\n", encoding="utf-8")
    (evals / "evals.json").write_text(
        '[{"id":"case-001","question":"Run the native case.","files":[]}]\n',
        encoding="utf-8",
    )
    (evals / "config.yml").write_text(
        f"schema_version: 1\nharbor:\n  task_source: native_harbor\ngrading:\n  mode: {grading_mode}\n",
        encoding="utf-8",
    )
    if grading_mode == "default_plus_custom":
        (evals / "grader.py").write_text("def grade(*args, **kwargs):\n    return 1\n", encoding="utf-8")
    (native_task / "instruction.md").write_text("Run the native case.\n", encoding="utf-8")
    authored_judge = f'{judge_alias} = "authored-native-judge"\n'
    judge_config = (
        f"[verifier]\ntimeout_sec = 180.0\n\n[verifier.env]\n{authored_judge}\n"
        if judge_scope == "task"
        else f'[[steps]]\nname = "step-one"\n\n[steps.verifier.env]\n{authored_judge}\n'
    )
    (native_task / "task.toml").write_text(
        'schema_version = "1.3"\n\n[task]\nname = "nvidia/case-001"\n\n'
        f'[metadata]\nentry_id = "case-001"\n\n{judge_config}[environment]\n',
        encoding="utf-8",
    )

    for name in ("LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL"):
        monkeypatch.delenv(name, raising=False)
    monkeypatch.setattr(runner, "resolve_llm_provider", lambda: _provider("nv_build"))
    monkeypatch.setattr(
        runner,
        "load_evals_config",
        lambda _path: (
            {"harbor": {"task_source": "native_harbor"}, "grading": {"mode": grading_mode}},
            None,
        ),
    )
    monkeypatch.setattr(runner, "_check_prerequisites", lambda **_kwargs: [])

    probed_models: list[str] = []

    def probe(selected_provider: ProviderConfig) -> runtime_preflight.ModelProbeResult:
        probed_models.append(selected_provider.model)
        return runtime_preflight.ModelProbeResult(
            True,
            selected_provider.provider,
            selected_provider.model,
            f"model {selected_provider.model} is available",
        )

    staged_tasks: list[Path] = []

    def launch(**kwargs):
        staged_tasks.append(kwargs["with_skill"] / "case-001")
        return []

    monkeypatch.setattr(runtime_preflight, "probe_model", probe)
    monkeypatch.setattr(runner, "_run_agent_pair", launch)
    monkeypatch.setattr(
        runner,
        "collect_harbor_results",
        lambda **_kwargs: {"execution_status": "complete", "execution_errors": [], "metrics": [], "agents": {}},
    )

    def render_report(_skill_path: Path, output_dir: Path, **_kwargs) -> Path:
        report = output_dir / "report.html"
        report.write_text("<html></html>\n", encoding="utf-8")
        return report

    monkeypatch.setattr(runner, "render_agent_eval_html_report", render_report)

    result = runner.run_harbor_eval(
        skill,
        ["opencode"],
        agent_models={"opencode": "nvidia/nvidia/nemotron-3-nano-30b-a3b"},
        output_dir=tmp_path / "results",
        env_mode="docker",
        grading_mode=grading_mode,
        skip_baseline=True,
        keep_harbor_jobs=True,
        agent_runtime_preflight=False,
    )

    assert result["execution_status"] == "failed"
    assert any("standard grading" in error and "environment control" in error for error in result["error"])
    assert result["run_config"]["task_source"] == "native_harbor"
    assert staged_tasks == []


def test_custom_only_native_verifier_placeholders_use_task_fallbacks_not_standard_override(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path,
) -> None:
    skill = tmp_path / "demo"
    evals = skill / "evals"
    native_task = evals / "harbor" / "case-001"
    native_task.mkdir(parents=True)
    (skill / "SKILL.md").write_text("# Demo\n", encoding="utf-8")
    (evals / "evals.json").write_text(
        '[{"id":"case-001","question":"Run the native case.","files":[]}]\n',
        encoding="utf-8",
    )
    (evals / "config.yml").write_text(
        "schema_version: 1\nharbor:\n  task_source: native_harbor\ngrading:\n  mode: custom_only\n",
        encoding="utf-8",
    )
    (evals / "grader.py").write_text("def grade(*args, **kwargs):\n    return 1\n", encoding="utf-8")
    (native_task / "instruction.md").write_text("Run the native case.\n", encoding="utf-8")
    (native_task / "task.toml").write_text(
        'schema_version = "1.3"\n\n[task]\nname = "nvidia/case-001"\n\n'
        '[metadata]\nentry_id = "case-001"\n\n'
        '[verifier.env]\nLLM_JUDGE_MODEL = "${LLM_JUDGE_MODEL:-task-legacy-model}"\n'
        'SKILL_EVAL_JUDGE_MODEL = "${SKILL_EVAL_JUDGE_MODEL:-task-canonical-model}"\n\n[environment]\n',
        encoding="utf-8",
    )
    launched: dict[str, object] = {}

    monkeypatch.setenv("LLM_JUDGE_MODEL", "standard-legacy-model")
    monkeypatch.setenv("SKILL_EVAL_JUDGE_MODEL", "standard-canonical-model")
    monkeypatch.setenv("LLM_JUDGE_FALLBACK_MODELS", "standard-fallback-one,standard-fallback-two")
    monkeypatch.setattr(runner, "resolve_llm_provider", lambda: _provider("nv_build"))
    monkeypatch.setattr(
        runner,
        "load_evals_config",
        lambda _path: ({"harbor": {"task_source": "native_harbor"}}, None),
    )
    monkeypatch.setattr(runner, "_check_prerequisites", lambda **_kwargs: [])

    def launch(**kwargs):
        launched.update(
            with_skill=kwargs["with_skill"],
            run_env=dict(kwargs["run_env"]),
            verifier_env=dict(kwargs["verifier_env"]),
        )
        return []

    monkeypatch.setattr(runner, "_run_agent_pair", launch)
    monkeypatch.setattr(
        runtime_preflight,
        "probe_model",
        lambda selected_provider: runtime_preflight.ModelProbeResult(
            True,
            selected_provider.provider,
            selected_provider.model,
            f"model {selected_provider.model} is available",
        ),
    )
    monkeypatch.setattr(
        runner,
        "collect_harbor_results",
        lambda **_kwargs: {"execution_status": "complete", "execution_errors": [], "metrics": [], "agents": {}},
    )
    monkeypatch.setattr(runner, "render_agent_eval_html_report", lambda *_args, **_kwargs: tmp_path / "report.html")

    result = runner.run_harbor_eval(
        skill,
        ["opencode"],
        agent_models={"opencode": "nvidia/nvidia/nemotron-3-nano-30b-a3b"},
        output_dir=tmp_path / "results",
        env_mode="docker",
        grading_mode="custom_only",
        skip_baseline=True,
        keep_harbor_jobs=True,
        agent_runtime_preflight=False,
    )

    assert "error" not in result
    assert result["run_config"]["judge"] == {"enabled": False}
    assert launched["verifier_env"] == {}
    task_config = tomllib.loads(
        (launched["with_skill"] / "case-001" / "task.toml").read_text(encoding="utf-8")  # type: ignore[operator]
    )
    authored_env = task_config["verifier"]["env"]
    run_env = launched["run_env"]
    assert "LLM_JUDGE_FALLBACK_MODELS" not in authored_env
    assert "LLM_JUDGE_FALLBACK_MODELS" not in run_env
    task_fallbacks = {
        "LLM_JUDGE_MODEL": "task-legacy-model",
        "SKILL_EVAL_JUDGE_MODEL": "task-canonical-model",
    }
    for name, fallback in task_fallbacks.items():
        assert name not in run_env
        assert authored_env[name] == f"${{{name}:-{fallback}}}"
        with monkeypatch.context() as harbor_process:
            harbor_process.setattr(os, "environ", dict(run_env))
            assert resolve_env_vars({name: authored_env[name]}) == {name: fallback}


def test_run_harbor_eval_rejects_provenance_key_inside_skill_before_creation(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path,
) -> None:
    skill = tmp_path / "demo"
    (skill / "evals").mkdir(parents=True)
    (skill / "evals" / "evals.json").write_text("[]\n", encoding="utf-8")
    key_path = skill / "private-output-provenance.key"
    monkeypatch.setenv("SKILLEVALUATOR_OUTPUT_PROVENANCE_KEY_FILE", str(key_path))
    monkeypatch.setattr(runner, "resolve_llm_provider", lambda: _provider("nv_build"))
    monkeypatch.setattr(
        runner,
        "load_evals_config",
        lambda _path: ({"harbor": {"task_source": "evals_json"}}, None),
    )
    monkeypatch.setattr(runner, "find_evals_file", lambda _path: skill / "evals" / "evals.json")
    monkeypatch.setattr(runner, "_check_prerequisites", lambda **_kwargs: [])
    monkeypatch.setattr(
        runtime_preflight,
        "probe_model",
        lambda selected_provider: runtime_preflight.ModelProbeResult(
            True,
            selected_provider.provider,
            selected_provider.model,
            f"model {selected_provider.model} is available",
        ),
    )

    result = runner.run_harbor_eval(
        skill,
        ["opencode"],
        output_dir=tmp_path / "results",
        env_mode="docker",
        agent_runtime_preflight=False,
    )

    assert "error" in result
    assert "provenance key must be outside" in result["error"][0]
    assert not key_path.exists()
    assert not (tmp_path / "results").exists()


def test_harbor_subprocess_environment_excludes_arbitrary_host_secrets(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path,
) -> None:
    host_environment = {
        "PATH": os.environ["PATH"],
        "HOME": str(tmp_path),
        "USERPROFILE": str(tmp_path),
        "TMPDIR": str(tmp_path / "tmp"),
        "LANG": "C.UTF-8",
        "DOCKER_HOST": "unix:///safe/docker.sock",
        "HOST_DATABASE_PASSWORD": "must-not-leak",
        "SSH_AUTH_SOCK": "/private/agent.sock",
        "ANTHROPIC_API_KEY": "unrelated-provider-secret",
        "E2B_API_KEY": "unselected-backend-secret",
    }
    monkeypatch.setattr(runner.os, "environ", host_environment)
    provider = _provider()
    provider_env = runner._provider_environment(provider)

    environment = runner._harbor_subprocess_environment(
        env_mode="docker",
        provider=provider,
        configured_runtime_env={"SERVICE_TOKEN": "declared-runtime-secret"},
        provider_env=provider_env,
    )

    assert environment == {
        "PATH": host_environment["PATH"],
        "HOME": str(tmp_path),
        "TMPDIR": str(tmp_path / "tmp"),
        "LANG": "C.UTF-8",
        "DOCKER_HOST": "unix:///safe/docker.sock",
        "SSH_AUTH_SOCK": "/private/agent.sock",
        "SERVICE_TOKEN": "declared-runtime-secret",
        "USERPROFILE": str(tmp_path),
        **provider_env,
    }


def test_harbor_subprocess_environment_includes_only_selected_backend_credentials(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(
        runner.os,
        "environ",
        {
            "PATH": "/usr/bin",
            "HOME": "/home/test",
            "E2B_API_KEY": "selected-key",
            "MODAL_TOKEN_ID": "unselected-id",
            "MODAL_TOKEN_SECRET": "unselected-secret",
        },
    )
    provider = _provider()

    environment = runner._harbor_subprocess_environment(
        env_mode="e2b",
        provider=provider,
        configured_runtime_env={},
        provider_env=runner._provider_environment(provider),
    )

    assert environment["E2B_API_KEY"] == "selected-key"
    assert "MODAL_TOKEN_ID" not in environment
    assert "MODAL_TOKEN_SECRET" not in environment


def test_harbor_subprocess_environment_preserves_trusted_proxy_and_tls_controls(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    network_environment = {
        "HTTP_PROXY": "http://proxy.example:8080",
        "HTTPS_PROXY": "https://proxy.example:8443",
        "NO_PROXY": "localhost,127.0.0.1",
        "ALL_PROXY": "socks5://proxy.example:1080",
        "http_proxy": "http://legacy-proxy.example:8080",
        "https_proxy": "https://legacy-proxy.example:8443",
        "no_proxy": "metadata.internal",
        "all_proxy": "socks5://legacy-proxy.example:1080",
        "SSL_CERT_FILE": "/etc/ssl/custom.pem",
        "SSL_CERT_DIR": "/etc/ssl/custom",
        "REQUESTS_CA_BUNDLE": "/etc/ssl/requests.pem",
        "CURL_CA_BUNDLE": "/etc/ssl/curl.pem",
    }
    monkeypatch.setattr(
        runner.os,
        "environ",
        {"PATH": "/usr/bin", "HOME": "/home/test", **network_environment, "UNRELATED_SECRET": "no"},
    )
    provider = _provider()

    environment = runner._harbor_subprocess_environment(
        env_mode="e2b",
        provider=provider,
        configured_runtime_env={},
        provider_env=runner._provider_environment(provider),
    )

    assert {name: environment[name] for name in network_environment} == network_environment
    assert "UNRELATED_SECRET" not in environment


def test_harbor_backend_environment_allowlist_covers_every_native_022_mode() -> None:
    expected = {
        "docker": {
            "COMPOSE_ANSI",
            "COMPOSE_HTTP_TIMEOUT",
            "COMPOSE_IGNORE_ORPHANS",
            "COMPOSE_PARALLEL_LIMIT",
            "COMPOSE_PROGRESS",
            "COMPOSE_STATUS_STDOUT",
            "DOCKER_API_VERSION",
            "DOCKER_AUTH_CONFIG",
            "DOCKER_CERT_PATH",
            "DOCKER_CONFIG",
            "DOCKER_CONTEXT",
            "DOCKER_CUSTOM_HEADERS",
            "DOCKER_DEFAULT_PLATFORM",
            "DOCKER_HOST",
            "DOCKER_TLS",
            "DOCKER_TLS_VERIFY",
            "SSH_AUTH_SOCK",
        },
        "daytona": {
            "DAYTONA_API_KEY",
            "DAYTONA_API_URL",
            "DAYTONA_HAPPY_EYEBALLS_DELAY",
            "DAYTONA_JWT_TOKEN",
            "DAYTONA_ORGANIZATION_ID",
            "DAYTONA_SERVER_URL",
            "DAYTONA_TARGET",
        },
        "e2b": {"E2B_API_KEY", "E2B_API_URL", "E2B_DOMAIN", "E2B_SANDBOX_URL"},
        "modal": {
            "MODAL_CONFIG_PATH",
            "MODAL_ENVIRONMENT",
            "MODAL_OVERRIDE_HEADERS",
            "MODAL_PROFILE",
            "MODAL_SERVER_URL",
            "MODAL_TOKEN_ID",
            "MODAL_TOKEN_SECRET",
        },
        "runloop": {"RUNLOOP_API_KEY", "RUNLOOP_BASE_URL", "RUNLOOP_CUSTOM_HEADERS"},
        "langsmith": {
            "LANGCHAIN_API_KEY",
            "LANGCHAIN_ENDPOINT",
            "LANGSMITH_API_KEY",
            "LANGSMITH_CONFIG_FILE",
            "LANGSMITH_ENDPOINT",
            "LANGSMITH_PROFILE",
            "LANGSMITH_SANDBOX_API_URL",
            "LANGSMITH_WORKSPACE_ID",
        },
        "ec2": {
            "AWS_ACCOUNT_ID",
            "AWS_ACCOUNT_ID_ENDPOINT_MODE",
            "AWS_ACCESS_KEY_ID",
            "AWS_AUTH_SCHEME_PREFERENCE",
            "AWS_CA_BUNDLE",
            "AWS_CONFIG_FILE",
            "AWS_CONTAINER_AUTHORIZATION_TOKEN",
            "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
            "AWS_CONTAINER_CREDENTIALS_FULL_URI",
            "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI",
            "AWS_CREDENTIAL_EXPIRATION",
            "AWS_CREDENTIAL_FILE",
            "AWS_CSM_CLIENT_ID",
            "AWS_CSM_ENABLED",
            "AWS_CSM_HOST",
            "AWS_CSM_PORT",
            "AWS_DATA_PATH",
            "AWS_DEFAULT_PROFILE",
            "AWS_DEFAULT_REGION",
            "AWS_DEFAULTS_MODE",
            "AWS_DISABLE_HOST_PREFIX_INJECTION",
            "AWS_DISABLE_REQUEST_COMPRESSION",
            "AWS_EC2_METADATA_DISABLED",
            "AWS_EC2_METADATA_SERVICE_ENDPOINT",
            "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE",
            "AWS_EC2_METADATA_V1_DISABLED",
            "AWS_ENDPOINT_DISCOVERY_ENABLED",
            "AWS_ENDPOINT_URL",
            "AWS_ENDPOINT_URL_EC2",
            "AWS_ENDPOINT_URL_SIGNIN",
            "AWS_ENDPOINT_URL_SSO",
            "AWS_ENDPOINT_URL_SSO_OIDC",
            "AWS_ENDPOINT_URL_STS",
            "AWS_EXECUTION_ENV",
            "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS",
            "AWS_IMDS_USE_IPV6",
            "AWS_LOGIN_CACHE_DIRECTORY",
            "AWS_MAX_ATTEMPTS",
            "AWS_METADATA_SERVICE_NUM_ATTEMPTS",
            "AWS_METADATA_SERVICE_TIMEOUT",
            "AWS_NEW_RETRIES_2026",
            "AWS_PROFILE",
            "AWS_REGION",
            "AWS_RETRY_MODE",
            "AWS_REQUEST_CHECKSUM_CALCULATION",
            "AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES",
            "AWS_RESPONSE_CHECKSUM_VALIDATION",
            "AWS_ROLE_ARN",
            "AWS_ROLE_SESSION_NAME",
            "AWS_SDK_LOAD_CONFIG",
            "AWS_SDK_UA_APP_ID",
            "AWS_SECRET_ACCESS_KEY",
            "AWS_SECURITY_TOKEN",
            "AWS_SESSION_TOKEN",
            "AWS_SHARED_CREDENTIALS_FILE",
            "AWS_SIGV4A_SIGNING_REGION_SET",
            "AWS_STS_REGIONAL_ENDPOINTS",
            "AWS_USE_DUALSTACK_ENDPOINT",
            "AWS_USE_FIPS_ENDPOINT",
            "AWS_WEB_IDENTITY_TOKEN_FILE",
            "BOTOCORE_TCP_KEEPALIVE",
            "SSH_AUTH_SOCK",
        },
        "gke": {
            "CLOUDSDK_ACTIVE_CONFIG_NAME",
            "CLOUDSDK_AUTH_ACCESS_TOKEN",
            "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE",
            "CLOUDSDK_AUTH_IMPERSONATE_SERVICE_ACCOUNT",
            "CLOUDSDK_CONFIG",
            "CLOUDSDK_CORE_ACCOUNT",
            "CLOUDSDK_CORE_PROJECT",
            "GCP_PROJECT",
            "GOOGLE_APPLICATION_CREDENTIALS",
            "GOOGLE_CLOUD_PROJECT",
            "GOOGLE_CLOUD_QUOTA_PROJECT",
            "KUBECONFIG",
            "SKILLEVALUATOR_GKE_ALLOW_WORKLOAD_IDENTITY",
            "SKILLEVALUATOR_GKE_AUTOPILOT",
            "SKILLEVALUATOR_GKE_METADATA_PROBE_IMAGE",
        },
        "ack": {
            "COMPOSE_ANSI",
            "COMPOSE_HTTP_TIMEOUT",
            "COMPOSE_IGNORE_ORPHANS",
            "COMPOSE_PARALLEL_LIMIT",
            "COMPOSE_PROGRESS",
            "COMPOSE_STATUS_STDOUT",
            "DOCKER_API_VERSION",
            "DOCKER_AUTH_CONFIG",
            "DOCKER_CERT_PATH",
            "DOCKER_CONFIG",
            "DOCKER_CONTEXT",
            "DOCKER_CUSTOM_HEADERS",
            "DOCKER_DEFAULT_PLATFORM",
            "DOCKER_HOST",
            "DOCKER_TLS",
            "DOCKER_TLS_VERIFY",
            "KUBECONFIG",
            "KUBERNETES_SERVICE_HOST",
            "KUBERNETES_SERVICE_PORT",
            "SSH_AUTH_SOCK",
        },
        "openshift": {"KUBECONFIG"},
        "novita": {
            "NOVITA_ACCESS_TOKEN",
            "NOVITA_API_KEY",
            "NOVITA_API_URL",
            "NOVITA_BASE_URL",
            "NOVITA_DOMAIN",
            "NOVITA_SANDBOX_URL",
        },
        "apple-container": set(),
        "singularity": {
            "APPTAINER_AUTHFILE",
            "APPTAINER_CONFIGDIR",
            "APPTAINER_DOCKER_PASSWORD",
            "APPTAINER_DOCKER_USERNAME",
            "SINGULARITY_AUTHFILE",
            "SINGULARITY_CONFIGDIR",
            "SINGULARITY_DOCKER_PASSWORD",
            "SINGULARITY_DOCKER_USERNAME",
        },
        "islo": {"ISLO_API_KEY", "ISLO_API_URL", "ISLO_COMPUTE_URL"},
        "tensorlake": {
            "TENSORLAKE_API_KEY",
            "TENSORLAKE_API_URL",
            "TENSORLAKE_ORGANIZATION_ID",
            "TENSORLAKE_PAT",
            "TENSORLAKE_PROJECT_ID",
            "TENSORLAKE_SANDBOX_PROXY_URL",
        },
        "cwsandbox": {"CWSANDBOX_API_KEY", "CWSANDBOX_BASE_URL"},
        "wandb": {"NETRC", "WANDB_API_KEY", "WANDB_BASE_URL", "WANDB_ENTITY", "WANDB_PROJECT"},
        "use-computer": {
            "USE_COMPUTER_API_KEY",
            "USE_COMPUTER_HOST",
            "USE_COMPUTER_SNAPSHOT",
            "USE_COMPUTER_VERSION",
        },
        "blaxel": {
            "BL_API_KEY",
            "BL_API_VERSION",
            "BL_CLIENT_CREDENTIALS",
            "BL_ENV",
            "BL_REGION",
            "BL_WORKSPACE",
        },
        "beam": {
            "API_HOST",
            "API_PORT",
            "BEAM_TOKEN",
            "GATEWAY_HOST",
            "GATEWAY_PORT",
            "INTERNAL_API_HOST",
            "INTERNAL_API_PORT",
            "REALTIME_HOST",
        },
        "skypilot": {
            "COMPOSE_ANSI",
            "COMPOSE_HTTP_TIMEOUT",
            "COMPOSE_IGNORE_ORPHANS",
            "COMPOSE_PARALLEL_LIMIT",
            "COMPOSE_PROGRESS",
            "COMPOSE_STATUS_STDOUT",
            "DOCKER_API_VERSION",
            "DOCKER_AUTH_CONFIG",
            "DOCKER_CERT_PATH",
            "DOCKER_CONFIG",
            "DOCKER_CONTEXT",
            "DOCKER_CUSTOM_HEADERS",
            "DOCKER_DEFAULT_PLATFORM",
            "DOCKER_HOST",
            "DOCKER_TLS",
            "DOCKER_TLS_VERIFY",
            "HARBOR_SKYPILOT_REGISTRY",
            "SKYPILOT_API_SERVER_ENDPOINT",
            "SKYPILOT_GLOBAL_CONFIG",
            "SKYPILOT_PROJECT_CONFIG",
            "SKYPILOT_SERVICE_ACCOUNT_TOKEN",
            "SSH_AUTH_SOCK",
        },
        "hyperbrowser": {
            "COMPOSE_ANSI",
            "COMPOSE_HTTP_TIMEOUT",
            "COMPOSE_IGNORE_ORPHANS",
            "COMPOSE_PARALLEL_LIMIT",
            "COMPOSE_PROGRESS",
            "COMPOSE_STATUS_STDOUT",
            "DOCKER_API_VERSION",
            "DOCKER_AUTH_CONFIG",
            "DOCKER_CERT_PATH",
            "DOCKER_CONFIG",
            "DOCKER_CONTEXT",
            "DOCKER_CUSTOM_HEADERS",
            "DOCKER_DEFAULT_PLATFORM",
            "DOCKER_HOST",
            "DOCKER_TLS",
            "DOCKER_TLS_VERIFY",
            "HYPERBROWSER_API_KEY",
            "HYPERBROWSER_BASE_URL",
            "SSH_AUTH_SOCK",
        },
        "vercel": {"VERCEL_OIDC_TOKEN", "VERCEL_PROJECT_ID", "VERCEL_TEAM_ID", "VERCEL_TOKEN"},
        "runta": {"RUNTA_CONFIG", "RUNTA_ENDPOINT", "RUNTA_TOKEN"},
        "mosaic": {
            "MAR_API_TOKEN",
            "MAR_CONFIG",
            "MAR_ENDPOINT",
            "MOSAIC_API_TOKEN",
            "MOSAIC_API_URL",
            "MOSAIC_CONFIG",
            "MOSAIC_REGISTRY_PASSWORD",
            "MOSAIC_REGISTRY_USERNAME",
            "MOSAIC_RETRIES",
        },
    }

    # Kata runs on the host's Docker engine, so it selects exactly Docker's variables.
    expected["kata"] = expected["docker"]

    assert set(expected) == HARBOR_NATIVE_ENV_MODES
    assert {mode: set(names) for mode, names in runner._HARBOR_ENV_MODE_VARS.items()} == expected


def test_direct_run_returns_configuration_error_for_cyclic_environment_kwargs(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    cyclic: dict[str, object] = {}
    cyclic["nested"] = cyclic
    monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
    monkeypatch.setattr(runner, "load_evals_config", lambda _path: ({}, None))

    result = runner._run_harbor_eval_impl(
        tmp_path,
        ["opencode"],
        environment_kwargs={"options": cyclic},
        _evaluator_skill_path=tmp_path,
    )

    assert result == {"error": ["Invalid --environment-kwarg: must not contain cyclic values"]}


@pytest.mark.parametrize("timeout_multiplier", [float("nan"), float("inf"), float("-inf")])
def test_direct_run_rejects_nonfinite_timeout_multiplier_before_creating_results(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    timeout_multiplier: float,
) -> None:
    monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
    monkeypatch.setattr(
        runner,
        "load_evals_config",
        lambda _path: ({"harbor": {"task_source": "evals_json"}}, None),
    )

    result = runner._run_harbor_eval_impl(
        tmp_path,
        ["opencode"],
        timeout_multiplier=timeout_multiplier,
        output_dir=tmp_path / "results",
        _evaluator_skill_path=tmp_path,
    )

    assert result == {"error": ["timeout_multiplier must be a finite number greater than 0"]}
    assert not (tmp_path / "results").exists()


def test_direct_run_rejects_overflowing_timeout_multiplier_before_creating_results(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
    monkeypatch.setattr(
        runner,
        "load_evals_config",
        lambda _path: ({"harbor": {"task_source": "evals_json"}}, None),
    )

    result = runner._run_harbor_eval_impl(
        tmp_path,
        ["opencode"],
        timeout_multiplier=10**1000,
        output_dir=tmp_path / "results",
        _evaluator_skill_path=tmp_path,
    )

    assert result == {"error": ["timeout_multiplier must be a finite number greater than 0"]}
    assert not (tmp_path / "results").exists()


def test_direct_run_rejects_finite_multiplier_that_overflows_default_timeouts(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
    monkeypatch.setattr(
        runner,
        "load_evals_config",
        lambda _path: ({"harbor": {"task_source": "evals_json"}}, None),
    )

    result = runner._run_harbor_eval_impl(
        tmp_path,
        ["opencode"],
        timeout_multiplier=1e308,
        output_dir=tmp_path / "results",
        _evaluator_skill_path=tmp_path,
    )

    assert result == {"error": ["timeout_multiplier must yield finite Harbor timeouts"]}
    assert not (tmp_path / "results").exists()


def test_direct_run_rejects_overflowing_pass_threshold_before_creating_results(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
    monkeypatch.setattr(
        runner,
        "load_evals_config",
        lambda _path: ({"harbor": {"task_source": "evals_json"}}, None),
    )

    result = runner._run_harbor_eval_impl(
        tmp_path,
        ["opencode"],
        pass_threshold=10**1000,
        output_dir=tmp_path / "results",
        _evaluator_skill_path=tmp_path,
    )

    assert result == {"error": ["pass_threshold must be a finite number between 0.0 and 1.0"]}
    assert not (tmp_path / "results").exists()


@pytest.mark.parametrize("setup_key", ["pre_agent_setup", "setup_commands"])
def test_paired_run_rejects_normalized_skill_owned_setup_before_environment_preflight(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    setup_key: str,
) -> None:
    skill = tmp_path / "skill"
    evals = skill / "evals"
    evals.mkdir(parents=True)
    (evals / "config.yml").write_text(
        f"schema_version: 1\nharbor:\n  {setup_key}: echo ready\n",
        encoding="utf-8",
    )
    monkeypatch.setattr(runner, "resolve_llm_provider", _provider)

    def unexpected_preflight(**_kwargs: object) -> list[str]:
        pytest.fail("paired skill-owned setup reached environment preflight")

    monkeypatch.setattr(runner, "_check_prerequisites", unexpected_preflight)

    result = runner._run_harbor_eval_impl(
        skill,
        ["opencode"],
        output_dir=tmp_path / "results",
        _evaluator_skill_path=skill,
    )

    assert result == {
        "error": [
            "harbor.pre_agent_setup/setup_commands cannot run in a paired evaluation; "
            "use --skip-baseline for a with-skill-only run"
        ]
    }
    assert not (tmp_path / "results").exists()


@pytest.mark.parametrize("setup_key", ["pre_agent_setup", "setup_commands"])
def test_skip_baseline_keeps_normalized_skill_owned_setup_enabled(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    setup_key: str,
) -> None:
    skill = tmp_path / "skill"
    evals = skill / "evals"
    evals.mkdir(parents=True)
    (evals / "config.yml").write_text(
        f"schema_version: 1\nharbor:\n  {setup_key}: echo ready\n",
        encoding="utf-8",
    )
    monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
    monkeypatch.setattr(runner, "_check_prerequisites", lambda **_kwargs: ["preflight sentinel"])

    result = runner._run_harbor_eval_impl(
        skill,
        ["opencode"],
        skip_baseline=True,
        output_dir=tmp_path / "results",
        _evaluator_skill_path=skill,
    )

    assert result == {"error": ["preflight sentinel"]}


@pytest.mark.parametrize(
    ("env_mode", "selected_name"),
    [
        ("ec2", "AWS_ACCESS_KEY_ID"),
        ("ec2", "SSH_AUTH_SOCK"),
        ("docker", "SSH_AUTH_SOCK"),
        ("modal", "MODAL_SERVER_URL"),
        ("gke", "CLOUDSDK_ACTIVE_CONFIG_NAME"),
        ("gke", "CLOUDSDK_CORE_PROJECT"),
        ("ack", "KUBECONFIG"),
        ("ack", "KUBERNETES_SERVICE_HOST"),
        ("ack", "KUBERNETES_SERVICE_PORT"),
        ("ack", "DOCKER_CONFIG"),
        ("openshift", "KUBECONFIG"),
        ("novita", "NOVITA_SANDBOX_URL"),
        ("singularity", "APPTAINER_AUTHFILE"),
        ("tensorlake", "TENSORLAKE_ORGANIZATION_ID"),
        ("blaxel", "BL_API_KEY"),
        ("beam", "BEAM_TOKEN"),
        ("beam", "API_HOST"),
        ("skypilot", "SKYPILOT_API_SERVER_ENDPOINT"),
        ("skypilot", "DOCKER_CONFIG"),
        ("hyperbrowser", "HYPERBROWSER_API_KEY"),
        ("hyperbrowser", "DOCKER_HOST"),
        ("wandb", "NETRC"),
        ("vercel", "VERCEL_OIDC_TOKEN"),
        ("runta", "RUNTA_TOKEN"),
        ("runta", "RUNTA_ENDPOINT"),
        ("mosaic", "MOSAIC_API_TOKEN"),
        ("mosaic", "MOSAIC_API_URL"),
        ("mosaic", "MOSAIC_CONFIG"),
        ("mosaic", "MAR_API_TOKEN"),
    ],
)
def test_new_harbor_022_backend_environment_is_selected_without_cross_backend_leakage(
    monkeypatch: pytest.MonkeyPatch,
    env_mode: str,
    selected_name: str,
) -> None:
    host_environment = {
        "PATH": "/usr/bin",
        "HOME": "/home/test",
        selected_name: "selected-value",
        "E2B_API_KEY": "unselected-value",
    }
    monkeypatch.setattr(runner.os, "environ", host_environment)
    provider = _provider()

    environment = runner._harbor_subprocess_environment(
        env_mode=env_mode,
        provider=provider,
        configured_runtime_env={},
        provider_env=runner._provider_environment(provider),
    )

    assert environment[selected_name] == "selected-value"
    assert "E2B_API_KEY" not in environment


def test_daytona_subprocess_environment_preserves_jwt_auth_pair(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(
        runner.os,
        "environ",
        {
            "PATH": "/usr/bin",
            "HOME": "/home/test",
            "DAYTONA_JWT_TOKEN": "jwt-token",
            "DAYTONA_ORGANIZATION_ID": "organization-id",
        },
    )
    provider = _provider()

    environment = runner._harbor_subprocess_environment(
        env_mode="daytona",
        provider=provider,
        configured_runtime_env={},
        provider_env=runner._provider_environment(provider),
    )

    assert environment["DAYTONA_JWT_TOKEN"] == "jwt-token"
    assert environment["DAYTONA_ORGANIZATION_ID"] == "organization-id"


def test_config_declared_substitution_is_resolved_without_leaking_source_variable(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(
        runner.os,
        "environ",
        {"PATH": "/usr/bin", "HOME": "/home/test", "HOST_AGENT_LABEL": "runtime-value", "UNRELATED_SECRET": "no"},
    )
    configured_runtime_env, errors = runner._resolve_runtime_env(
        {"AGENT_LABEL": "${HOST_AGENT_LABEL}"},
        env_mode="docker",
    )
    provider = _provider()

    environment = runner._harbor_subprocess_environment(
        env_mode="docker",
        provider=provider,
        configured_runtime_env=configured_runtime_env,
        provider_env=runner._provider_environment(provider),
    )

    assert errors == []
    assert environment["AGENT_LABEL"] == "runtime-value"
    assert "HOST_AGENT_LABEL" not in environment
    assert "UNRELATED_SECRET" not in environment


def test_runtime_env_rejects_host_process_control_names() -> None:
    unsafe_names = (
        "PATH",
        "PYTHONPATH",
        "HOME",
        "USERPROFILE",
        "XDG_CONFIG_HOME",
        "CLAUDE_CONFIG_DIR",
        "CLAUDE_CODE_DISABLE_POLICY_SKILLS",
        "CODEX_HOME",
        "GEMINI_CLI_HOME",
        "OPENCODE_CONFIG_DIR",
        "LD_PRELOAD",
        "DYLD_INSERT_LIBRARIES",
        "BASH_ENV",
        "BASH_FUNC_hidden%%",
        "NODE_OPTIONS",
        "LLM_JUDGE_FALLBACK_MODELS",
        "DOCKER_HOST",
        "COMPOSE_FILE",
        "HARBOR_HOME",
        "OTEL_EXPORTER_OTLP_ENDPOINT",
        "HTTPS_PROXY",
        "E2B_API_KEY",
        "AWS_CONFIG_FILE",
        "AWS_ENDPOINT_URL",
        "AWS_PROFILE",
        "AWS_WEB_IDENTITY_TOKEN_FILE",
    )

    for name in unsafe_names:
        resolved, errors = runner._resolve_runtime_env(
            {name: "attacker-controlled"},
            env_mode="docker",
        )
        assert name not in resolved
        assert any(name in error and "host process" in error for error in errors)


def test_bedrock_subprocess_environment_keeps_only_explicit_aws_provider_values(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(
        runner.os,
        "environ",
        {
            "PATH": "/usr/bin",
            "HOME": "/home/test",
            "AWS_ACCESS_KEY_ID": "access-key",
            "AWS_SECRET_ACCESS_KEY": "secret-key",
            "AWS_SESSION_TOKEN": "session-token",
            "AWS_PROFILE": "evaluation",
            "AWS_BEARER_TOKEN_BEDROCK": "bearer-token",
            "AWS_ROLE_ARN": "arn:aws:iam::123456789012:role/evaluator",
            "AWS_ROLE_SESSION_NAME": "evaluation-session",
            "AWS_WEB_IDENTITY_TOKEN_FILE": "/var/run/secrets/token",
            "OTHER_CLOUD_SECRET": "must-not-leak",
        },
    )
    provider = _provider("bedrock")
    provider_env = runner._provider_environment(provider)

    environment = runner._harbor_subprocess_environment(
        env_mode="docker",
        provider=provider,
        configured_runtime_env={},
        provider_env=provider_env,
    )

    assert provider_env["AWS_ACCESS_KEY_ID"] == "access-key"
    assert provider_env["AWS_BEARER_TOKEN_BEDROCK"] == "bearer-token"
    assert provider_env["AWS_WEB_IDENTITY_TOKEN_FILE"] == "/var/run/secrets/token"
    assert {
        "AWS_ACCESS_KEY_ID",
        "AWS_BEARER_TOKEN_BEDROCK",
        "AWS_WEB_IDENTITY_TOKEN_FILE",
    }.issubset(_verifier_env_vars(provider_env))
    assert environment["AWS_ACCESS_KEY_ID"] == "access-key"
    assert environment["AWS_SECRET_ACCESS_KEY"] == "secret-key"
    assert environment["AWS_SESSION_TOKEN"] == "session-token"
    assert environment["AWS_PROFILE"] == "evaluation"
    assert environment["AWS_BEARER_TOKEN_BEDROCK"] == "bearer-token"
    assert environment["AWS_ROLE_ARN"].endswith(":role/evaluator")
    assert environment["AWS_ROLE_SESSION_NAME"] == "evaluation-session"
    assert environment["AWS_WEB_IDENTITY_TOKEN_FILE"] == "/var/run/secrets/token"
    assert "OTHER_CLOUD_SECRET" not in environment


def test_bedrock_host_credential_allowlist_is_staged_for_the_verifier() -> None:
    expected = set(runner._BEDROCK_HOST_ENV_VARS) | {"AWS_REGION"}

    assert set(_verifier_env_vars(dict.fromkeys(expected, "configured"))) == expected


def test_local_nvidia_provider_preserves_explicit_codex_credentials(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path,
) -> None:
    monkeypatch.setattr(
        runner.os,
        "environ",
        {
            "PATH": "/usr/bin",
            "HOME": str(tmp_path),
            "SKILLEVALUATOR_RUNTIME_DIR": str(tmp_path / "runtimes"),
        },
    )
    provider = _provider("nv_build")

    environment = runner._harbor_subprocess_environment(
        env_mode="local",
        provider=provider,
        configured_runtime_env={
            "OPENAI_API_KEY": "codex-key",
            "OPENAI_BASE_URL": "https://api.openai.com/v1",
        },
        provider_env=runner._provider_environment(provider),
    )

    assert environment["NVIDIA_API_KEY"] == "provider-key"
    assert environment["OPENAI_API_KEY"] == "codex-key"
    assert environment["OPENAI_BASE_URL"] == "https://api.openai.com/v1"


def test_resolve_agent_runtime_plan_preserves_refreshed_adc_token(monkeypatch: pytest.MonkeyPatch) -> None:
    """Verify _resolve_agent_runtime_plan preserves refreshed ADC token and does not overwrite with expired token."""
    base_url = "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi"
    provider = ProviderConfig(
        provider="openai",
        model="google/gemini-3.8-flash",
        api_key="expired-token",
        base_url=base_url,
        litellm_model="openai/google/gemini-3.8-flash",
        credential_env="ADC",
        base_url_env="OPENAI_BASE_URL",
    )
    monkeypatch.setattr(runner, "_get_google_access_token", lambda **_kw: "fresh-token")
    monkeypatch.setattr(runner.os, "environ", {"PATH": "/usr/bin"})

    plans = runner._resolve_agent_runtime_plan(
        provider=provider,
        agents=["opencode", "codex"],
        models={
            "opencode": "openai/google/gemini-3.8-flash",
            "codex": "google/gemini-3.8-flash",
        },
        configured_runtime_env={},
        env_mode="docker",
    )

    for agent in ("opencode", "codex"):
        plan = plans[agent]
        assert plan.subprocess_env["OPENAI_API_KEY"] == "fresh-token"
        assert plan.provider.api_key == "fresh-token"
        assert plan.subprocess_env["OPENAI_BASE_URL"] == base_url


def test_run_harbor_reissues_adc_token_for_bounded_jobs(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
    """Verify _run_harbor refreshes ADC token in child env without leaking secrets to argv and sets a bounded timeout."""
    base_url = "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi"
    captured_command: list[str] = []
    captured_env: dict[str, str] = {}
    captured_timeout: float | None = None

    def mock_run(command, *args, **kwargs):
        nonlocal captured_timeout
        captured_command.extend(command)
        captured_env.update(kwargs.get("env") or {})
        captured_timeout = kwargs.get("timeout_seconds")
        return runner._BoundedHarborProcessResult(returncode=0, output_tail="", output_exceeded=False)

    monkeypatch.setattr(runner, "_run_bounded_harbor_process", mock_run)
    monkeypatch.setattr(runner, "_validate_harbor_job_result", lambda *_args, **_kwargs: (True, "success"))
    monkeypatch.setattr(runner, "_get_google_access_token", lambda **_kw: "fresh-reissued-token")

    run_env = {
        "OPENAI_API_KEY": "stale-initial-token",
        "OPENAI_BASE_URL": base_url,
        "SKILL_EVAL_LLM_CREDENTIAL_SOURCE": "ADC",
        "PATH": "/usr/bin",
    }
    verifier_env = {"LLM_JUDGE_MODEL": "${LLM_JUDGE_MODEL}"}

    ok, _detail = runner._run_harbor(
        dataset=tmp_path / "dataset",
        agent="opencode",
        job_name="test-job",
        env_mode="docker",
        model="google/gemini-3.8-flash",
        jobs_dir=tmp_path / "jobs",
        run_env=run_env,
        n_attempts=1,
        n_concurrent=1,
        timeout_multiplier=1.0,
        override_cpus=None,
        override_memory_mb=None,
        override_storage_mb=None,
        verifier_env=verifier_env,
    )

    assert ok is True
    rendered_argv = " ".join(captured_command)
    assert "fresh-reissued-token" not in rendered_argv
    assert "stale-initial-token" not in rendered_argv
    assert "OPENAI_API_KEY=" not in rendered_argv
    assert captured_env.get("OPENAI_API_KEY") == "fresh-reissued-token"
    assert captured_timeout is not None and 0 < captured_timeout < 3600


def test_run_harbor_preserves_explicit_vertex_credential(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    """Preserve explicit operator credentials on Vertex URLs without overwriting from host ADC."""
    base_url = "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi"
    captured_env: dict[str, str] = {}
    captured_timeout: float | None = None

    def mock_run(command, *args, **kwargs):
        nonlocal captured_timeout
        captured_env.update(kwargs.get("env") or {})
        captured_timeout = kwargs.get("timeout_seconds")
        return runner._BoundedHarborProcessResult(returncode=0, output_tail="", output_exceeded=False)

    monkeypatch.setattr(runner, "_run_bounded_harbor_process", mock_run)
    monkeypatch.setattr(runner, "_validate_harbor_job_result", lambda *_args, **_kwargs: (True, "success"))
    monkeypatch.setattr("skillevaluator.provider_config._get_google_access_token", lambda **_kw: "ambient-host-adc")

    explicit_env = {
        "OPENAI_API_KEY": "explicit-operator-key",
        "OPENAI_BASE_URL": base_url,
        "PATH": "/usr/bin",
    }
    ok, _detail = runner._run_harbor(
        dataset=tmp_path / "dataset",
        agent="opencode",
        job_name="explicit-job",
        env_mode="docker",
        model="google/gemini-3.8-flash",
        jobs_dir=tmp_path / "jobs",
        run_env=explicit_env,
        n_attempts=1,
        n_concurrent=1,
        timeout_multiplier=1.0,
        override_cpus=None,
        override_memory_mb=None,
        override_storage_mb=None,
    )
    assert ok is True
    assert captured_env.get("OPENAI_API_KEY") == "explicit-operator-key"
    assert captured_timeout is None


@pytest.mark.parametrize(
    "verifier_env",
    [
        {"OPENAI_API_KEY": "raw-secret-token"},
        {"ANTHROPIC_API_KEY": "sk-ant-raw-secret"},
        {"SKILL_EVAL_LLM_CREDENTIAL_SOURCE": "ADC"},
    ],
)
def test_build_harbor_run_command_rejects_raw_verifier_env_secrets(
    tmp_path: Path,
    verifier_env: dict[str, str],
) -> None:
    """Reject raw secrets or operator-owned values in verifier_env without ${VAR} placeholder indirection."""
    with pytest.raises(ValueError, match="Sensitive key detected in verifier_env"):
        runner.build_harbor_run_command(
            dataset_path=tmp_path / "dataset",
            agent="opencode",
            job_name="bad-verifier-env",
            env_mode="docker",
            verifier_env=verifier_env,
        )


def test_run_harbor_enforces_adc_job_timeout_and_stages_credential_source(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    """Enforce bounded ADC timeout on non-GKE jobs and stage SKILL_EVAL_LLM_CREDENTIAL_SOURCE for the verifier."""
    base_url = "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi"
    adc_provider = ProviderConfig(
        provider="openai-compatible",
        model="google/gemini-3.8-flash",
        api_key="initial-adc-token",
        base_url=base_url,
        litellm_model="openai/google/gemini-3.8-flash",
        credential_env="ADC",
        base_url_env="SKILL_EVAL_LLM_BASE_URL",
    )
    monkeypatch.setattr("skillevaluator.provider_config._get_google_access_token", lambda **_kw: "fresh-adc-token")
    monkeypatch.setattr(runner, "_get_google_access_token", lambda **_kw: "fresh-adc-token")

    provider_env = runner._provider_environment(adc_provider)
    assert provider_env.get("SKILL_EVAL_LLM_CREDENTIAL_SOURCE") == "ADC"
    assert "SKILL_EVAL_LLM_CREDENTIAL_SOURCE" in _verifier_env_vars(provider_env)

    def timeout_run(command, *args, **kwargs):
        raise runner._HarborRunTimeoutError(f"timed out after {kwargs.get('timeout_seconds', 3300)}s")

    monkeypatch.setattr(runner, "_run_bounded_harbor_process", timeout_run)
    adc_env = {
        "OPENAI_API_KEY": "initial-adc-token",
        "OPENAI_BASE_URL": base_url,
        "SKILL_EVAL_LLM_CREDENTIAL_SOURCE": "ADC",
        "PATH": "/usr/bin",
    }
    ok_timeout, detail_timeout = runner._run_harbor(
        dataset=tmp_path / "dataset",
        agent="opencode",
        job_name="timeout-job",
        env_mode="docker",
        model="google/gemini-3.8-flash",
        jobs_dir=tmp_path / "jobs",
        run_env=adc_env,
        n_attempts=1,
        n_concurrent=1,
        timeout_multiplier=1.0,
        override_cpus=None,
        override_memory_mb=None,
        override_storage_mb=None,
    )
    assert ok_timeout is False
    assert "ADC token lifetime" in detail_timeout


def test_run_harbor_caps_configured_adc_timeout_and_rejects_near_expiry(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    """Cap SKILLEVALUATOR_VERTEX_ADC_JOB_TIMEOUT_SEC to remaining token ceiling and reject <= 30s tokens."""
    import time as time_mod

    base_url = "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi"
    fixed_now = 1_700_000_000.0
    monkeypatch.setattr(time_mod, "time", lambda: fixed_now)
    monkeypatch.setattr("skillevaluator.provider_config._get_google_access_token", lambda **_kw: None)
    monkeypatch.setattr(runner, "_get_google_access_token", lambda **_kw: None)

    captured_timeouts: list[float | None] = []

    def fake_run(command, *args, **kwargs):
        captured_timeouts.append(kwargs.get("timeout_seconds"))
        return runner._BoundedHarborProcessResult(returncode=0, output_tail="", output_exceeded=False)

    monkeypatch.setattr(runner, "_run_bounded_harbor_process", fake_run)
    monkeypatch.setattr(runner, "_validate_harbor_job_result", lambda *_args, **_kwargs: (True, "success"))

    # 1. 600s token + SKILLEVALUATOR_VERTEX_ADC_JOB_TIMEOUT_SEC=5000 -> capped at 540s
    monkeypatch.setenv(runner.VERTEX_ADC_JOB_TIMEOUT_ENV, "5000")
    adc_env_600 = {
        "OPENAI_API_KEY": "short-lived-adc-token",
        "OPENAI_BASE_URL": base_url,
        "SKILL_EVAL_LLM_CREDENTIAL_SOURCE": "ADC",
        "SKILL_EVAL_LLM_CREDENTIAL_EXPIRY": str(fixed_now + 600.0),
        "PATH": "/usr/bin",
    }
    ok_capped, _ = runner._run_harbor(
        dataset=tmp_path / "dataset",
        agent="opencode",
        job_name="capped-job",
        env_mode="docker",
        model="google/gemini-3.8-flash",
        jobs_dir=tmp_path / "jobs",
        run_env=adc_env_600,
        n_attempts=1,
        n_concurrent=1,
        timeout_multiplier=1.0,
        override_cpus=None,
        override_memory_mb=None,
        override_storage_mb=None,
    )
    assert ok_capped is True
    assert captured_timeouts[-1] == pytest.approx(540.0)

    # 2. 600s token + SKILLEVALUATOR_VERTEX_ADC_JOB_TIMEOUT_SEC=300 -> honors 300s
    monkeypatch.setenv(runner.VERTEX_ADC_JOB_TIMEOUT_ENV, "300")
    ok_smaller, _ = runner._run_harbor(
        dataset=tmp_path / "dataset",
        agent="opencode",
        job_name="smaller-job",
        env_mode="docker",
        model="google/gemini-3.8-flash",
        jobs_dir=tmp_path / "jobs",
        run_env=adc_env_600,
        n_attempts=1,
        n_concurrent=1,
        timeout_multiplier=1.0,
        override_cpus=None,
        override_memory_mb=None,
        override_storage_mb=None,
    )
    assert ok_smaller is True
    assert captured_timeouts[-1] == pytest.approx(300.0)

    # 3. Malformed or non-positive SKILLEVALUATOR_VERTEX_ADC_JOB_TIMEOUT_SEC falls back to 540s ceiling
    for bad_override in ("invalid", "-10", "nan"):
        monkeypatch.setenv(runner.VERTEX_ADC_JOB_TIMEOUT_ENV, bad_override)
        ok_bad_override, _ = runner._run_harbor(
            dataset=tmp_path / "dataset",
            agent="opencode",
            job_name="bad-override-job",
            env_mode="docker",
            model="google/gemini-3.8-flash",
            jobs_dir=tmp_path / "jobs",
            run_env=adc_env_600,
            n_attempts=1,
            n_concurrent=1,
            timeout_multiplier=1.0,
            override_cpus=None,
            override_memory_mb=None,
            override_storage_mb=None,
        )
        assert ok_bad_override is True
        assert captured_timeouts[-1] == pytest.approx(540.0)

    # 4. 20s remaining (<= 30s safety floor) fails closed before launching Harbor
    monkeypatch.delenv(runner.VERTEX_ADC_JOB_TIMEOUT_ENV, raising=False)
    adc_env_near_expiry = {
        **adc_env_600,
        "SKILL_EVAL_LLM_CREDENTIAL_EXPIRY": str(fixed_now + 20.0),
    }
    ok_near, detail_near = runner._run_harbor(
        dataset=tmp_path / "dataset",
        agent="opencode",
        job_name="near-expiry-job",
        env_mode="docker",
        model="google/gemini-3.8-flash",
        jobs_dir=tmp_path / "jobs",
        run_env=adc_env_near_expiry,
        n_attempts=1,
        n_concurrent=1,
        timeout_multiplier=1.0,
        override_cpus=None,
        override_memory_mb=None,
        override_storage_mb=None,
    )
    assert ok_near is False
    assert "only 20.0s remaining" in detail_near
    assert len(captured_timeouts) == 5


def test_harbor_subprocess_environment_forwards_adc_vars_only_to_non_local_modes(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    """Forward GOOGLE_APPLICATION_CREDENTIALS and CLOUDSDK_CONFIG to docker/gke Harbor host processes but not local."""
    adc_file = tmp_path / "custom_adc.json"
    adc_file.write_text("{}", encoding="utf-8")
    gcloud_dir = tmp_path / "gcloud-cfg"
    gcloud_dir.mkdir()
    monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", str(adc_file))
    monkeypatch.setenv("CLOUDSDK_CONFIG", str(gcloud_dir))

    adc_provider = ProviderConfig(
        provider="openai-compatible",
        model="google/gemini-3.8-flash",
        api_key="adc-token",
        base_url="https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi",
        litellm_model="openai/google/gemini-3.8-flash",
        credential_env="ADC",
        base_url_env="SKILL_EVAL_LLM_BASE_URL",
    )
    provider_env = {"SKILL_EVAL_LLM_CREDENTIAL_SOURCE": "ADC", "OPENAI_API_KEY": "adc-token"}

    docker_env = runner._harbor_subprocess_environment(
        env_mode="docker",
        provider=adc_provider,
        configured_runtime_env={},
        provider_env=provider_env,
        agent="opencode",
        agent_model="openai/google/gemini-3.8-flash",
    )
    assert docker_env.get("GOOGLE_APPLICATION_CREDENTIALS") == str(adc_file)
    assert docker_env.get("CLOUDSDK_CONFIG") == str(gcloud_dir)

    local_env = runner._harbor_subprocess_environment(
        env_mode="local",
        provider=adc_provider,
        configured_runtime_env={},
        provider_env=provider_env,
        agent="opencode",
        agent_model="openai/google/gemini-3.8-flash",
    )
    assert "GOOGLE_APPLICATION_CREDENTIALS" not in local_env
    assert "CLOUDSDK_CONFIG" not in local_env


def test_harbor_subprocess_environment_forwards_gke_probe_and_autopilot_vars(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    """Forward SKILLEVALUATOR_GKE_METADATA_PROBE_IMAGE and SKILLEVALUATOR_GKE_AUTOPILOT in gke mode and allow --ek autopilot."""
    from skillevaluator.tier3.evals_config import _GKE_INFRASTRUCTURE_KWARGS, encode_environment_kwarg

    monkeypatch.setenv("SKILLEVALUATOR_GKE_METADATA_PROBE_IMAGE", "us-docker.pkg.dev/my-proj/repo/python:3.12-alpine")
    monkeypatch.setenv("SKILLEVALUATOR_GKE_AUTOPILOT", "true")

    provider = _provider("openai")
    provider_env = runner._provider_environment(provider)

    gke_env = runner._harbor_subprocess_environment(
        env_mode="gke",
        provider=provider,
        configured_runtime_env={},
        provider_env=provider_env,
        agent="opencode",
        agent_model="openai/test-model",
    )
    assert gke_env.get("SKILLEVALUATOR_GKE_METADATA_PROBE_IMAGE") == "us-docker.pkg.dev/my-proj/repo/python:3.12-alpine"
    assert gke_env.get("SKILLEVALUATOR_GKE_AUTOPILOT") == "true"

    docker_env = runner._harbor_subprocess_environment(
        env_mode="docker",
        provider=provider,
        configured_runtime_env={},
        provider_env=provider_env,
        agent="opencode",
        agent_model="openai/test-model",
    )
    assert "SKILLEVALUATOR_GKE_METADATA_PROBE_IMAGE" not in docker_env
    assert "SKILLEVALUATOR_GKE_AUTOPILOT" not in docker_env

    assert "autopilot" in _GKE_INFRASTRUCTURE_KWARGS
    cmd = runner.build_harbor_run_command(
        dataset_path=tmp_path / "dataset",
        agent="opencode",
        job_name="gke-job",
        env_mode="gke",
        environment_kwargs={"autopilot": "true", "cluster_name": "c"},
    )
    assert encode_environment_kwarg("autopilot", "true") in cmd