Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

33.9 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Docker subprocess argv must never contain evaluator or agent credentials."""

from __future__ import annotations

import asyncio
import contextlib
import contextvars
import hashlib
import importlib
import importlib.util
import io
import os
import subprocess
import sys
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from types import SimpleNamespace

import pytest


def _initialize_harbor_context(environment: object) -> None:
    environment._output_callbacks = contextvars.ContextVar("test_docker_output_callbacks", default=())
    environment._exec_env_overlays = contextvars.ContextVar("test_docker_exec_env_overlays", default=())


def test_secure_docker_exec_streams_environment_without_host_file_or_argv_values(
    monkeypatch,
    tmp_path: Path,
) -> None:
    module_name = "skillevaluator.tier3.harbor.secure_docker_environment"
    assert importlib.util.find_spec(module_name) is not None, "secure Docker environment module is missing"
    module = importlib.import_module(module_name)
    environment = object.__new__(module.SkillEvaluatorSecureDockerEnvironment)
    _initialize_harbor_context(environment)
    environment._persistent_env = {"PERSISTENT_TOKEN": "persistent-secret-value"}
    environment.default_user = "1000"
    environment.task_env_config = SimpleNamespace(workdir="/workspace")
    environment._platform = SimpleNamespace(exec_shell_args=lambda command: ["bash", "-c", command])

    del tmp_path
    uploaded: list[tuple[str, str]] = []
    docker_commands: list[list[str]] = []
    handoff_payloads: list[str] = []

    async def fake_upload_file(source_path: Path | str, target_path: str) -> None:
        uploaded.append((Path(source_path).read_text(encoding="utf-8"), target_path))

    async def fake_run(
        command: list[str],
        check: bool = True,
        timeout_sec: int | None = None,
        stdin_data: bytes | None = None,
        on_output: object | None = None,
        *,
        additional_secret_values: set[str] | None = None,
        exact_secret_values: set[str] | None = None,
        stop_main_on_interrupt: bool = False,
    ):
        del check, timeout_sec, on_output, exact_secret_values
        assert stop_main_on_interrupt is ("if ! ." in " ".join(command))
        if stdin_data is not None:
            handoff_payloads.append(stdin_data.decode("utf-8"))
            assert additional_secret_values is not None
        docker_commands.append(command)
        return SimpleNamespace(stdout="ok", stderr=None, return_code=0)

    monkeypatch.setattr(environment, "upload_file", fake_upload_file)
    monkeypatch.setattr(environment, "_run_docker_compose_command", fake_run)

    secret = "nvidia-secret-value-for-test"
    result = asyncio.run(
        environment.exec(
            "printf done",
            env={"NVIDIA_API_KEY": secret, "VISIBLE_SETTING": "enabled"},
        )
    )

    assert result.return_code == 0
    assert uploaded == []
    assert len(handoff_payloads) == 1
    assert secret in handoff_payloads[0]
    assert "persistent-secret-value" in handoff_payloads[0]
    rendered_argv = "\n".join("\0".join(command) for command in docker_commands)
    assert secret not in rendered_argv
    assert "persistent-secret-value" not in rendered_argv
    assert "NVIDIA_API_KEY=" not in rendered_argv
    assert all("-e" not in command for command in docker_commands)
    assert any("rm -f" in part for command in docker_commands for part in command)


def test_secure_docker_exec_redacts_persistent_and_per_call_credentials_from_all_output(
    monkeypatch,
    tmp_path: Path,
) -> None:
    module = importlib.import_module("skillevaluator.tier3.harbor.secure_docker_environment")
    environment = object.__new__(module.SkillEvaluatorSecureDockerEnvironment)
    _initialize_harbor_context(environment)
    persistent_secret = "persistent-credential-for-redaction"
    per_call_secret = "per-call-credential-for-redaction"
    environment._persistent_env = {"PERSISTENT_TOKEN": persistent_secret}
    environment.default_user = "1000"
    environment.task_env_config = SimpleNamespace(workdir="/workspace")
    environment._platform = SimpleNamespace(exec_shell_args=lambda command: ["bash", "-c", command])
    environment.session_id = "secure-redaction-test"
    environment.environment_name = "secure-redaction-test"
    environment.environment_dir = tmp_path
    environment._resources_compose_path = None
    environment._env_compose_path = None
    environment._mounts_compose_path = None
    environment._stream_compose_path = None
    environment._use_prebuilt = True
    environment._is_windows_container = False
    environment.extra_docker_compose_paths = []
    environment._network_policy = SimpleNamespace(network_mode="public")
    environment._enable_egress_control = False
    environment._egress_control_services_compose_path = None
    environment._compose_env_vars = lambda **_kwargs: {"PATH": "/usr/bin"}
    environment._compose_infra_env_vars = dict
    environment._windows_container_name = None

    async def fake_upload_file(_source_path: Path | str, _target_path: str) -> None:
        return None

    class FakeInput:
        def write(self, _data: bytes) -> None:
            return None

        async def drain(self) -> None:
            return None

        def close(self) -> None:
            return None

        async def wait_closed(self) -> None:
            return None

    class FakeOutput:
        def __init__(self, payload: bytes) -> None:
            self._payload = payload

        async def read(self, _size: int) -> bytes:
            payload, self._payload = self._payload, b""
            return payload

    class FakeProcess:
        returncode = 0

        def __init__(self, *, expose_secrets: bool) -> None:
            payload = b""
            if expose_secrets:
                output = f"stdout {persistent_secret} {per_call_secret}".encode()
                error = f"stderr {per_call_secret} {persistent_secret}".encode()
                payload = output + error
            self.stdin = FakeInput()
            self.stdout = FakeOutput(payload)

        async def wait(self) -> int:
            return self.returncode

    async def create_subprocess(*args: object, **_kwargs: object) -> FakeProcess:
        rendered = " ".join(str(arg) for arg in args)
        return FakeProcess(expose_secrets="if ! ." in rendered)

    monkeypatch.setattr(environment, "upload_file", fake_upload_file)
    monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)

    result = asyncio.run(environment.exec("printf done", env={"NVIDIA_API_KEY": per_call_secret}))

    rendered = f"{result.stdout}\n{result.stderr}"
    assert persistent_secret not in rendered
    assert per_call_secret not in rendered
    assert rendered.count("[REDACTED]") == 4


def test_nvidia_build_docker_command_uses_secure_environment_and_bridge() -> None:
    from skillevaluator.tier3.harbor import runner

    agent_import_path = "skillevaluator.tier3.harbor.local_agents:SkillEvaluatorNvidiaBuildCodex"

    command = runner.build_harbor_run_command(
        dataset_path="/tmp/dataset",
        agent="codex",
        job_name="secure-build",
        env_mode="docker",
        agent_import_path=agent_import_path,
    )

    assert "--agent-import-path" not in command
    assert "--environment-import-path" not in command
    assert "-a" not in command
    assert command[command.index("--agent") + 1] == agent_import_path
    assert command[command.index("--env") + 1] == runner.SECURE_DOCKER_ENV_IMPORT_PATH


def test_secure_docker_exec_cleans_remote_handoff_when_streaming_reports_failure(
    monkeypatch,
    tmp_path: Path,
) -> None:
    del tmp_path
    module = importlib.import_module("skillevaluator.tier3.harbor.secure_docker_environment")
    environment = object.__new__(module.SkillEvaluatorSecureDockerEnvironment)
    _initialize_harbor_context(environment)
    environment._persistent_env = {}
    environment.default_user = "1000"
    environment.task_env_config = SimpleNamespace(workdir="/workspace")
    environment._platform = SimpleNamespace(exec_shell_args=lambda command: ["bash", "-c", command])
    removed: list[str] = []

    async def stream_then_fail(
        _command: list[str],
        _check: bool = True,
        _timeout_sec: int | None = None,
        stdin_data: bytes | None = None,
        _on_output: object | None = None,
        **_kwargs,
    ) -> SimpleNamespace:
        if stdin_data is not None:
            raise ConnectionError("docker exec disconnected after creating the file")
        return SimpleNamespace(stdout="", stderr=None, return_code=0)

    async def remove_handoff(remote_path: str) -> None:
        removed.append(remote_path)

    monkeypatch.setattr(environment, "_run_docker_compose_command", stream_then_fail)
    monkeypatch.setattr(environment, "_remove_handoff", remove_handoff)

    with pytest.raises(ConnectionError, match="docker exec disconnected"):
        asyncio.run(environment.exec("true", env={"NVIDIA_API_KEY": "secret-for-test"}))

    assert len(removed) == 1


def test_secure_docker_exec_repeated_cancellation_does_not_interrupt_handoff_cleanup(
    monkeypatch,
) -> None:
    module = importlib.import_module("skillevaluator.tier3.harbor.secure_docker_environment")
    environment = object.__new__(module.SkillEvaluatorSecureDockerEnvironment)
    _initialize_harbor_context(environment)
    environment._persistent_env = {}
    environment.default_user = "1000"
    environment.task_env_config = SimpleNamespace(workdir="/workspace")
    environment._platform = SimpleNamespace(exec_shell_args=lambda command: ["bash", "-c", command])

    async def run_cancelled() -> bool:
        upload_started = asyncio.Event()
        cleanup_started = asyncio.Event()
        release_cleanup = asyncio.Event()
        cleanup_finished = False

        async def blocked_stream(
            _command: list[str],
            _check: bool = True,
            _timeout_sec: int | None = None,
            stdin_data: bytes | None = None,
            _on_output: object | None = None,
            **_kwargs,
        ) -> SimpleNamespace:
            if stdin_data is not None:
                upload_started.set()
                await asyncio.Event().wait()
            return SimpleNamespace(stdout="", stderr=None, return_code=0)

        async def remove_handoff(_remote_path: str) -> None:
            nonlocal cleanup_finished
            cleanup_started.set()
            await release_cleanup.wait()
            cleanup_finished = True

        monkeypatch.setattr(environment, "_run_docker_compose_command", blocked_stream)
        monkeypatch.setattr(environment, "_remove_handoff", remove_handoff)

        task = asyncio.create_task(environment.exec("true", env={"NVIDIA_API_KEY": "secret-for-test"}))
        await asyncio.wait_for(upload_started.wait(), timeout=1)
        task.cancel()
        await asyncio.wait_for(cleanup_started.wait(), timeout=1)
        task.cancel()
        release_cleanup.set()
        with pytest.raises(asyncio.CancelledError):
            await asyncio.wait_for(task, timeout=1)
        return cleanup_finished

    assert asyncio.run(run_cancelled()) is True


def test_secure_docker_exec_fails_closed_when_final_secret_cleanup_fails(monkeypatch) -> None:
    module = importlib.import_module("skillevaluator.tier3.harbor.secure_docker_environment")
    environment = object.__new__(module.SkillEvaluatorSecureDockerEnvironment)
    _initialize_harbor_context(environment)
    environment._persistent_env = {}
    environment.default_user = "1000"
    environment.task_env_config = SimpleNamespace(workdir="/workspace")
    environment._platform = SimpleNamespace(exec_shell_args=lambda command: ["bash", "-c", command])

    async def fake_upload_file(_source_path: Path | str, _target_path: str) -> None:
        return None

    async def fake_run(
        command: list[str],
        check: bool = True,
        timeout_sec: int | None = None,
        stdin_data: bytes | None = None,
        on_output: object | None = None,
        *,
        additional_secret_values: set[str] | None = None,
        exact_secret_values: set[str] | None = None,
        stop_main_on_interrupt: bool = False,
    ):
        del (
            check,
            timeout_sec,
            stdin_data,
            on_output,
            additional_secret_values,
            exact_secret_values,
        )
        assert stop_main_on_interrupt is ("if ! ." in " ".join(command))
        return SimpleNamespace(stdout="ok", stderr=None, return_code=0)

    async def failed_cleanup(_remote_path: str) -> None:
        raise ConnectionError("container cleanup unavailable")

    monkeypatch.setattr(environment, "upload_file", fake_upload_file)
    monkeypatch.setattr(environment, "_run_docker_compose_command", fake_run)
    monkeypatch.setattr(environment, "_remove_handoff", failed_cleanup)

    with pytest.raises(RuntimeError, match="could not confirm removal"):
        asyncio.run(environment.exec("true", env={"NVIDIA_API_KEY": "secret-for-test"}))


def test_secure_docker_remove_handoff_rejects_nonzero_docker_result(monkeypatch) -> None:
    module = importlib.import_module("skillevaluator.tier3.harbor.secure_docker_environment")
    environment = object.__new__(module.SkillEvaluatorSecureDockerEnvironment)

    async def failed_run(command: list[str], check: bool = True, timeout_sec: int | None = None):
        del command, check, timeout_sec
        return SimpleNamespace(stdout="", stderr="rm failed", return_code=1)

    monkeypatch.setattr(environment, "_run_docker_compose_command", failed_run)

    with pytest.raises(RuntimeError, match="Docker environment handoff removal failed"):
        asyncio.run(environment._remove_handoff("/tmp/secret-handoff"))


def test_harbor_subprocess_receives_nvidia_key_only_over_stdin(monkeypatch, tmp_path: Path) -> None:
    from skillevaluator.tier3.harbor import runner

    secret = "nvidia-real-secret-value-for-test"

    def reject_temporary_directory(*_args, **_kwargs):
        raise AssertionError("NVIDIA key handoff must not create a temporary directory")

    monkeypatch.setattr(runner.tempfile, "TemporaryDirectory", reject_temporary_directory)
    launch_dirs: list[Path] = []

    @contextlib.contextmanager
    def recording_launch_cwd():
        # The evaluator-owned launch directory is allowed, but must stay empty.
        launch_dir = tmp_path / f"launch-{len(launch_dirs)}"
        launch_dir.mkdir()
        launch_dirs.append(launch_dir)
        yield launch_dir

    monkeypatch.setattr(runner, "_harbor_launch_cwd", recording_launch_cwd)

    def fake_run(command, **kwargs):
        environment = kwargs["env"]
        assert kwargs["cwd"] in launch_dirs
        assert not any(Path(kwargs["cwd"]).iterdir())
        assert secret not in command
        assert secret not in environment.values()
        assert environment["NVIDIA_API_KEY"] == runner._NVIDIA_BUILD_STDIN_SENTINEL
        assert environment[runner._NVIDIA_BUILD_KEY_STDIN_ENV] == "1"
        assert runner._NVIDIA_BUILD_KEY_FILE_ENV not in environment
        assert kwargs["stdin_text"] == secret
        return runner._BoundedHarborProcessResult(returncode=0, output_tail="", output_exceeded=False)

    monkeypatch.setattr(runner, "_run_bounded_harbor_process", fake_run)
    monkeypatch.setattr(runner, "_validate_harbor_job_result", lambda *_args, **_kwargs: (True, ""))

    ok, detail = runner._run_harbor(
        dataset=tmp_path / "dataset",
        agent="opencode",
        job_name="secure-build",
        env_mode="docker",
        model="nvidia/nvidia/nemotron-3-nano-30b-a3b",
        jobs_dir=tmp_path / "jobs",
        run_env={
            "SKILL_EVAL_LLM_PROVIDER": "nv_build",
            "NVIDIA_API_KEY": secret,
        },
        n_attempts=1,
        n_concurrent=1,
        timeout_multiplier=1.0,
        override_cpus=None,
        override_memory_mb=None,
        override_storage_mb=None,
    )

    assert (ok, detail) == (True, "")
    assert launch_dirs
    assert all(not any(launch_dir.iterdir()) for launch_dir in launch_dirs)
    handoff = runner._nvidia_build_key_handoff(
        {"SKILL_EVAL_LLM_PROVIDER": "nv_build", "NVIDIA_API_KEY": secret},
        env_mode="docker",
    )
    assert secret not in repr(handoff)

    other_secret = "non-nvidia-provider-secret-value-for-test"
    other_handoff = runner._nvidia_build_key_handoff(
        {"SKILL_EVAL_LLM_PROVIDER": "openai", "OPENAI_API_KEY": other_secret},
        env_mode="docker",
    )
    assert other_handoff.subprocess_env["OPENAI_API_KEY"] == other_secret
    assert other_secret not in repr(other_handoff)


def test_harbor_subprocess_redacts_stdin_key_from_early_failure(monkeypatch, tmp_path: Path) -> None:
    from skillevaluator.tier3.harbor import runner

    secret = "nvidia-real-secret-value-for-test"

    def fake_run(command, **kwargs):
        assert kwargs["stdin_text"] == secret
        assert secret not in kwargs["env"].values()
        return runner._BoundedHarborProcessResult(
            returncode=1,
            output_tail=f"provider rejected {secret}",
            output_exceeded=False,
        )

    monkeypatch.setattr(runner, "_run_bounded_harbor_process", fake_run)

    ok, detail = runner._run_harbor(
        dataset=tmp_path / "dataset",
        agent="opencode",
        job_name="secure-build",
        env_mode="docker",
        model="nvidia/nvidia/nemotron-3-nano-30b-a3b",
        jobs_dir=tmp_path / "jobs",
        run_env={"SKILL_EVAL_LLM_PROVIDER": "nv_build", "NVIDIA_API_KEY": secret},
        n_attempts=1,
        n_concurrent=1,
        timeout_multiplier=1.0,
        override_cpus=None,
        override_memory_mb=None,
        override_storage_mb=None,
    )

    assert ok is False
    assert secret not in detail
    assert "redacted" in detail.lower()


def test_stdin_handoff_is_shared_by_docker_environment_and_nvidia_agent(monkeypatch) -> None:
    from skillevaluator.tier3.harbor import local_agents, runner, secure_docker_environment, sensitive_stdin

    secret = "nvidia-real-secret-value-for-test"
    monkeypatch.setattr(sensitive_stdin._nvidia_build_key_cache, "value", sensitive_stdin._UNSET)
    monkeypatch.setattr(sensitive_stdin.sys, "stdin", io.StringIO(secret))
    monkeypatch.setenv("NVIDIA_API_KEY", runner._NVIDIA_BUILD_STDIN_SENTINEL)
    monkeypatch.setenv(runner._NVIDIA_BUILD_KEY_STDIN_ENV, "1")

    resolved = secure_docker_environment._host_handoff_environment(
        {"NVIDIA_API_KEY": runner._NVIDIA_BUILD_STDIN_SENTINEL}
    )
    agent_key = local_agents.SkillEvaluatorNvidiaBuildCodex._resolve_bridge_api_key()

    assert resolved["NVIDIA_API_KEY"] == secret
    assert agent_key == secret
    assert sensitive_stdin.sys.stdin.read() == ""


def test_secure_docker_preflight_consumes_stdin_key_before_docker_child(monkeypatch) -> None:
    from harbor.environments.docker.docker import DockerEnvironment

    from skillevaluator.tier3.harbor import runner, secure_docker_environment, sensitive_stdin

    secret = "nvidia-real-secret-value-for-test"
    observed: list[str] = []
    monkeypatch.setattr(sensitive_stdin._nvidia_build_key_cache, "value", sensitive_stdin._UNSET)
    monkeypatch.setattr(sensitive_stdin.sys, "stdin", io.StringIO(secret))
    monkeypatch.setenv("NVIDIA_API_KEY", runner._NVIDIA_BUILD_STDIN_SENTINEL)
    monkeypatch.setenv(runner._NVIDIA_BUILD_KEY_STDIN_ENV, "1")

    def docker_preflight(_cls) -> None:
        assert sensitive_stdin._nvidia_build_key_cache.value == secret
        observed.append(secret)
        assert sensitive_stdin.sys.stdin.read() == ""

    monkeypatch.setattr(DockerEnvironment, "preflight", classmethod(docker_preflight))

    secure_docker_environment.SkillEvaluatorSecureDockerEnvironment.preflight()

    assert observed == [secret]


def test_stdin_handoff_fails_closed_without_marker(monkeypatch) -> None:
    from skillevaluator.tier3.harbor import runner, secure_docker_environment, sensitive_stdin

    monkeypatch.setattr(sensitive_stdin._nvidia_build_key_cache, "value", sensitive_stdin._UNSET)
    monkeypatch.setattr(sensitive_stdin.sys, "stdin", io.StringIO("must-not-be-read"))
    monkeypatch.delenv(runner._NVIDIA_BUILD_KEY_STDIN_ENV, raising=False)

    with pytest.raises(RuntimeError, match="SKILLEVALUATOR_NVIDIA_API_KEY_STDIN"):
        secure_docker_environment._host_handoff_environment({"NVIDIA_API_KEY": runner._NVIDIA_BUILD_STDIN_SENTINEL})

    assert sensitive_stdin.sys.stdin.read() == "must-not-be-read"


def test_non_handoff_environment_does_not_consume_stdin(monkeypatch) -> None:
    from skillevaluator.tier3.harbor import secure_docker_environment, sensitive_stdin

    monkeypatch.setattr(sensitive_stdin._nvidia_build_key_cache, "value", sensitive_stdin._UNSET)
    monkeypatch.setattr(sensitive_stdin.sys, "stdin", io.StringIO("unrelated-input"))

    resolved = secure_docker_environment._host_handoff_environment({"NVIDIA_API_KEY": "ordinary-key"})

    assert resolved == {"NVIDIA_API_KEY": "ordinary-key"}
    assert sensitive_stdin.sys.stdin.read() == "unrelated-input"


def test_stdin_handoff_is_read_once_under_concurrency(monkeypatch) -> None:
    from skillevaluator.tier3.harbor import runner, sensitive_stdin

    class CountingInput(io.StringIO):
        reads = 0

        def read(self, *args, **kwargs):
            self.reads += 1
            return super().read(*args, **kwargs)

    secret = "nvidia-real-secret-value-for-test"
    source = CountingInput(secret)
    monkeypatch.setattr(sensitive_stdin._nvidia_build_key_cache, "value", sensitive_stdin._UNSET)
    monkeypatch.setattr(sensitive_stdin.sys, "stdin", source)
    monkeypatch.setenv(runner._NVIDIA_BUILD_KEY_STDIN_ENV, "1")

    with ThreadPoolExecutor(max_workers=8) as executor:
        results = list(executor.map(lambda _index: sensitive_stdin.read_nvidia_build_key_from_stdin(), range(32)))

    assert results == [secret] * 32
    assert source.reads == 1


def test_real_child_process_receives_nvidia_key_only_through_stdin() -> None:
    from skillevaluator.tier3.harbor import runner

    secret = "nvidia-real-secret-value-for-test"
    expected_digest = hashlib.sha256(secret.encode("utf-8")).hexdigest()
    child = (
        "import hashlib; "
        "from skillevaluator.tier3.harbor.sensitive_stdin import read_nvidia_build_key_from_stdin; "
        "print(hashlib.sha256(read_nvidia_build_key_from_stdin().encode('utf-8')).hexdigest())"
    )
    environment = dict(os.environ)
    environment["NVIDIA_API_KEY"] = runner._NVIDIA_BUILD_STDIN_SENTINEL
    environment[runner._NVIDIA_BUILD_KEY_STDIN_ENV] = "1"
    environment.pop(runner._NVIDIA_BUILD_KEY_FILE_ENV, None)

    completed = subprocess.run(
        [sys.executable, "-c", child],
        input=secret,
        text=True,
        capture_output=True,
        env=environment,
        check=True,
    )

    assert completed.stdout.strip() == expected_digest
    assert completed.stderr == ""
    assert secret not in child
    assert secret not in environment.values()
    assert secret not in completed.stdout


def test_empty_stdin_handoff_fails_before_docker_preflight(monkeypatch) -> None:
    from harbor.environments.docker.docker import DockerEnvironment

    from skillevaluator.tier3.harbor import runner, secure_docker_environment, sensitive_stdin

    docker_preflight_called = False
    monkeypatch.setattr(sensitive_stdin._nvidia_build_key_cache, "value", sensitive_stdin._UNSET)
    monkeypatch.setattr(sensitive_stdin.sys, "stdin", io.StringIO(""))
    monkeypatch.setenv("NVIDIA_API_KEY", runner._NVIDIA_BUILD_STDIN_SENTINEL)
    monkeypatch.setenv(runner._NVIDIA_BUILD_KEY_STDIN_ENV, "1")

    def docker_preflight(_cls) -> None:
        nonlocal docker_preflight_called
        docker_preflight_called = True

    monkeypatch.setattr(DockerEnvironment, "preflight", classmethod(docker_preflight))

    with pytest.raises(RuntimeError, match="stdin handoff is empty"):
        secure_docker_environment.SkillEvaluatorSecureDockerEnvironment.preflight()

    assert docker_preflight_called is False


def test_secure_docker_exec_refreshes_adc_token_per_trial_and_verifier_exec(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Refresh host ADC token on every container exec() boundary so multi-trial Docker verifiers never use expired tokens."""
    from skillevaluator.tier3.harbor import secure_docker_environment

    environment = object.__new__(secure_docker_environment.SkillEvaluatorSecureDockerEnvironment)
    _initialize_harbor_context(environment)
    environment._persistent_env = {}
    environment.default_user = "1000"
    environment.task_env_config = SimpleNamespace(workdir="/workspace")
    environment._platform = SimpleNamespace(exec_shell_args=lambda command: ["bash", "-c", command])

    handoff_payloads: list[str] = []
    docker_commands: list[list[str]] = []

    async def fake_run(
        command: list[str],
        check: bool = True,
        timeout_sec: int | None = None,
        stdin_data: bytes | None = None,
        on_output: object | None = None,
        **_kwargs,
    ):
        del check, timeout_sec, on_output
        if stdin_data is not None:
            handoff_payloads.append(stdin_data.decode("utf-8"))
        docker_commands.append(command)
        return SimpleNamespace(stdout="ok", stderr=None, return_code=0)

    monkeypatch.setattr(environment, "_run_docker_compose_command", fake_run)

    tokens = iter(["rotated-adc-token-trial-1", "rotated-adc-token-trial-2"])
    monkeypatch.setattr(secure_docker_environment, "_get_google_access_token", lambda **_kw: next(tokens))

    vertex_url = "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi"

    # Trial 1 verifier exec (after initial launch token expired)
    asyncio.run(
        environment.exec(
            "bash /tests/test.sh",
            env={
                "OPENAI_API_KEY": "expired-launch-token",
                "OPENAI_BASE_URL": vertex_url,
                "SKILL_EVAL_LLM_CREDENTIAL_SOURCE": "ADC",
            },
        )
    )
    # Trial 2 verifier exec (later in the same Harbor job after Trial 1 token also expired)
    asyncio.run(
        environment.exec(
            "bash /tests/test.sh",
            env={
                "OPENAI_API_KEY": "expired-launch-token",
                "OPENAI_BASE_URL": vertex_url,
                "SKILL_EVAL_LLM_CREDENTIAL_SOURCE": "ADC",
            },
        )
    )
    # Explicit operator credential exec (must NOT be replaced by host ADC)
    asyncio.run(
        environment.exec(
            "bash /tests/test.sh",
            env={
                "OPENAI_API_KEY": "explicit-operator-vertex-key",
                "OPENAI_BASE_URL": vertex_url,
            },
        )
    )

    assert len(handoff_payloads) == 3
    assert "rotated-adc-token-trial-1" in handoff_payloads[0]
    assert "expired-launch-token" not in handoff_payloads[0]
    assert "rotated-adc-token-trial-2" in handoff_payloads[1]
    assert "expired-launch-token" not in handoff_payloads[1]
    assert "explicit-operator-vertex-key" in handoff_payloads[2]

    rendered_argv = "\n".join("\0".join(cmd) for cmd in docker_commands)
    assert "rotated-adc-token-trial-1" not in rendered_argv
    assert "rotated-adc-token-trial-2" not in rendered_argv


def test_short_expiry_adc_docker_handoff_and_failed_refresh_fails_closed(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    """Preserve custom ADC sources for Docker host refresh, bound timeout below short token expiry, and fail closed on refresh failure."""
    import json
    import types
    from datetime import UTC, datetime, timedelta

    from skillevaluator.provider_config import resolve_llm_provider
    from skillevaluator.tier3.harbor import runner, secure_docker_environment

    adc_file = tmp_path / "external_account_adc.json"
    adc_file.write_text(
        json.dumps(
            {
                "type": "external_account",
                "audience": "//iam.googleapis.com/projects/123456/locations/global/workloadIdentityPools/p/providers/pr",
                "subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
                "token_url": "https://sts.googleapis.com/v1/token",
                "service_account_impersonation_url": (
                    "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/"
                    "[email protected]:generateAccessToken"
                ),
                "service_account_impersonation": {"token_lifetime_seconds": 600},
                "credential_source": {"file": str(tmp_path / "subject_token.txt")},
            }
        ),
        encoding="utf-8",
    )
    cloudsdk_dir = tmp_path / "gcloud_config"
    cloudsdk_dir.mkdir()

    minted_counter = 0

    class _FakeCredentials:
        def __init__(self, lifetime_seconds: int) -> None:
            self._lifetime_seconds = lifetime_seconds
            self.token: str | None = None
            self.expiry: datetime | None = None

        def refresh(self, _request: object) -> None:
            nonlocal minted_counter
            minted_counter += 1
            self.token = f"short-lived-adc-token-{minted_counter}"
            # Match google.auth's naive UTC datetime expiry convention
            self.expiry = datetime.now(UTC).replace(tzinfo=None) + timedelta(seconds=self._lifetime_seconds)

    def fake_google_auth_default(scopes: object = None):
        del scopes
        cred_path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", "").strip()
        if not cred_path or not Path(cred_path).is_file():
            raise RuntimeError("ADC file could not be discovered from GOOGLE_APPLICATION_CREDENTIALS")
        payload = json.loads(Path(cred_path).read_text(encoding="utf-8"))
        lifetime = int(payload.get("service_account_impersonation", {}).get("token_lifetime_seconds", 3600))
        return _FakeCredentials(lifetime), "demo-project"

    google_mod = types.ModuleType("google")
    auth_mod = types.ModuleType("google.auth")
    transport_mod = types.ModuleType("google.auth.transport")
    requests_mod = types.ModuleType("google.auth.transport.requests")
    auth_mod.default = fake_google_auth_default  # type: ignore[attr-defined]
    requests_mod.Request = object  # type: ignore[attr-defined]
    google_mod.auth = auth_mod  # type: ignore[attr-defined]
    auth_mod.transport = transport_mod  # type: ignore[attr-defined]
    transport_mod.requests = requests_mod  # type: ignore[attr-defined]

    monkeypatch.setitem(sys.modules, "google", google_mod)
    monkeypatch.setitem(sys.modules, "google.auth", auth_mod)
    monkeypatch.setitem(sys.modules, "google.auth.transport", transport_mod)
    monkeypatch.setitem(sys.modules, "google.auth.transport.requests", requests_mod)
    monkeypatch.setattr("shutil.which", lambda _cmd: None)

    vertex_url = "https://aiplatform.googleapis.com/v1beta1/projects/demo-project/locations/global/endpoints/openapi"
    host_env = {
        "PATH": "/usr/bin",
        "HOME": str(tmp_path),
        "SKILL_EVAL_LLM_PROVIDER": "openai",
        "OPENAI_BASE_URL": vertex_url,
        "SKILL_EVAL_LLM_MODEL": "google/gemini-3.8-flash",
        "GOOGLE_APPLICATION_CREDENTIALS": str(adc_file),
        "CLOUDSDK_CONFIG": str(cloudsdk_dir),
    }
    monkeypatch.setattr(os, "environ", dict(host_env))

    provider = resolve_llm_provider()
    assert provider.api_key == "short-lived-adc-token-1"
    assert provider.credential_env == "ADC"
    assert provider.credential_expiry is not None

    plans = runner._resolve_agent_runtime_plan(
        provider=provider,
        agents=["opencode"],
        models={"opencode": "openai/google/gemini-3.8-flash"},
        configured_runtime_env={},
        env_mode="docker",
    )
    docker_subprocess_env = dict(plans["opencode"].subprocess_env)
    assert docker_subprocess_env.get("GOOGLE_APPLICATION_CREDENTIALS") == str(adc_file)
    assert docker_subprocess_env.get("CLOUDSDK_CONFIG") == str(cloudsdk_dir)
    assert "GOOGLE_APPLICATION_CREDENTIALS" not in plans["opencode"].staged_env
    assert "CLOUDSDK_CONFIG" not in plans["opencode"].staged_env

    # Verify _run_harbor enforces a deadline strictly below the 600s token lifetime (e.g. ~540s, not 3300s)
    captured_timeout: float | None = None

    def fake_harbor_run(command, *args, **kwargs):
        del command, args
        nonlocal captured_timeout
        captured_timeout = kwargs.get("timeout_seconds")
        return runner._BoundedHarborProcessResult(returncode=0, output_tail="", output_exceeded=False)

    monkeypatch.setattr(runner, "_run_bounded_harbor_process", fake_harbor_run)
    monkeypatch.setattr(runner, "_validate_harbor_job_result", lambda *_a, **_kw: (True, "ok"))

    ok, _detail = runner._run_harbor(
        dataset=tmp_path / "dataset",
        agent="opencode",
        job_name="short-adc-job",
        env_mode="docker",
        model="google/gemini-3.8-flash",
        jobs_dir=tmp_path / "jobs",
        run_env=docker_subprocess_env,
        n_attempts=1,
        n_concurrent=1,
        timeout_multiplier=1.0,
        override_cpus=None,
        override_memory_mb=None,
        override_storage_mb=None,
    )
    assert ok is True
    assert captured_timeout is not None
    assert 500.0 <= captured_timeout <= 545.0

    # Simulate the trusted Harbor Docker host subprocess executing verifier handoff
    monkeypatch.setattr(os, "environ", dict(docker_subprocess_env))
    verifier_input = {
        "OPENAI_API_KEY": docker_subprocess_env["OPENAI_API_KEY"],
        "OPENAI_BASE_URL": vertex_url,
        "SKILL_EVAL_LLM_CREDENTIAL_SOURCE": "ADC",
    }
    handed_off = secure_docker_environment._host_handoff_environment(verifier_input)
    assert handed_off["OPENAI_API_KEY"] != docker_subprocess_env["OPENAI_API_KEY"]
    assert handed_off["OPENAI_API_KEY"].startswith("short-lived-adc-token-")
    assert "GOOGLE_APPLICATION_CREDENTIALS" not in handed_off
    assert "CLOUDSDK_CONFIG" not in handed_off
    assert "SKILL_EVAL_LLM_CREDENTIAL_EXPIRY" not in handed_off

    # Missing-source / failed-refresh case: must fail closed instead of reusing the prior/expired token
    adc_file.unlink()
    with pytest.raises(RuntimeError, match="Failed to refresh Google ADC access token"):
        secure_docker_environment._host_handoff_environment(verifier_input)