Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

237 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Local mode (`--env-mode local`) wiring."""

from __future__ import annotations

import asyncio
import contextlib
import contextvars
import errno
import json
import logging
import os
import re
import selectors
import shlex
import signal
import socket
import subprocess
import sys
import tempfile
import threading
import time
import tomllib
import traceback
from pathlib import Path
from types import SimpleNamespace
from urllib.parse import urlsplit

import pytest
from harbor.agents.installed.opencode import OpenCode
from harbor.environments.base import BaseEnvironment
from harbor.models.task.config import EnvironmentConfig
from harbor.models.trial.paths import TrialPaths

from skillevaluator.provider_config import ProviderConfig
from skillevaluator.tier3.harbor import (
    DOCKER_AGENT_IMPORT_PATHS,
    ENV_MODE_LOCAL,
    HARBOR_ENV_MODES,
    HARBOR_NATIVE_ENV_MODES,
    LOCAL_AGENT_IMPORT_PATHS,
    LOCAL_ENV_IMPORT_PATH,
    local_sandbox,
)
from skillevaluator.tier3.harbor import stream_redaction as stream_redaction_module
from skillevaluator.tier3.harbor.local_agents import (
    NVIDIA_BUILD_AGENT_IMPORT_PATHS,
    SkillEvaluatorLocalClaudeCode,
    SkillEvaluatorLocalOpenCode,
    SkillEvaluatorNvidiaBuildClaudeCode,
    SkillEvaluatorNvidiaBuildCodex,
)
from skillevaluator.tier3.harbor.local_environment import SkillEvaluatorLocalEnvironment
from skillevaluator.tier3.harbor.local_runtime import ensure_local_runtimes, validate_local_agents
from skillevaluator.tier3.harbor.runner import (
    _check_prerequisites,
    _harbor_subprocess_environment,
    _local_agent_credentials,
    build_harbor_run_command,
)
from skillevaluator.tier3.harbor.secret_redaction import redact_secrets_in_log_line
from skillevaluator.tier3.harbor.secure_docker_environment import SECURE_DOCKER_ENV_IMPORT_PATH
from skillevaluator.tier3.harbor.stream_redaction import (
    CommandOutputLimitError,
    StreamingLogRedactor,
    StreamingSecretRedactor,
    _StreamingKnownPatternRedactor,
)

_NATIVE_WINDOWS_LOCAL_REASON = "native Windows local mode requires WSL2; these checks exercise the POSIX backend"


class _NoopScopedExecEnvironment:
    @contextlib.contextmanager
    def scoped_exec_env(self, _env: dict[str, str]):
        yield


class _LocalCallbackBaseError(BaseException):
    pass


def _local_environment(
    tmp_path: Path, *, persistent_env: dict[str, str] | None = None
) -> SkillEvaluatorLocalEnvironment:
    environment = object.__new__(SkillEvaluatorLocalEnvironment)
    environment._runtime_root = tmp_path / "runtime"
    environment._runtime_agent = "opencode"
    environment._root = tmp_path / "run"
    environment._workspace = environment._root / "workspace"
    environment._tests = environment._root / "tests"
    environment._solution = environment._root / "solution"
    environment._installed_agent = environment._root / "installed-agent"
    environment._tmp = environment._root / "tmp"
    environment._home = environment._root / "home"
    environment._sandbox_mode = "off"
    environment._allow_net = False
    environment._inherit_agent_keys = False
    environment._strict_reads = False
    environment._active_processes = {}
    environment._active_process_secret_values = {}
    environment._pending_creations = set()
    environment._creation_secret_values = {}
    environment._creation_cleanups = {}
    environment._creation_cleanup_errors = []
    environment._stop_requested = False
    environment._persistent_env = persistent_env or {}
    environment._output_callbacks = contextvars.ContextVar("test_local_output_callbacks", default=())
    environment._exec_env_overlays = contextvars.ContextVar("test_local_exec_env_overlays", default=())
    environment._sandbox = local_sandbox.Sandbox(local_sandbox.SandboxPlan("none", "advisory-only", "test"))
    environment.trial_paths = type(
        "TrialPaths",
        (),
        {
            "trial_dir": tmp_path / "trial",
            "agent_dir": tmp_path / "trial" / "agent",
            "user_agent_dir": tmp_path / "trial" / "user-agent",
            "verifier_dir": tmp_path / "trial" / "verifier",
            "artifacts_dir": tmp_path / "trial" / "artifacts",
            "reward_json_path": tmp_path / "trial" / "verifier" / "reward.json",
            "reward_text_path": tmp_path / "trial" / "verifier" / "reward.txt",
        },
    )()
    environment.logger = logging.getLogger("test-local-environment")
    for path in (
        environment._workspace,
        environment._tests,
        environment._solution,
        environment._installed_agent,
        environment._tmp,
        environment._home,
        environment.trial_paths.agent_dir,
        environment.trial_paths.verifier_dir,
        environment.trial_paths.artifacts_dir,
    ):
        path.mkdir(parents=True, exist_ok=True)
    return environment


def _initialized_local_environment(
    tmp_path: Path,
    *,
    persistent_env: dict[str, str] | None = None,
) -> SkillEvaluatorLocalEnvironment:
    environment_dir = tmp_path / "environment"
    environment_dir.mkdir()
    runtime_root = tmp_path / "runtime"
    runtime_root.mkdir()
    return SkillEvaluatorLocalEnvironment(
        environment_dir=environment_dir,
        environment_name="local-streaming-test",
        session_id="local-streaming-test",
        trial_paths=TrialPaths(tmp_path / "trial"),
        task_env_config=EnvironmentConfig(),
        runtime_agent="opencode",
        runtime_root=str(runtime_root),
        sandbox_mode="off",
        allow_net=False,
        persistent_env=persistent_env,
    )


def _assert_process_exits(pid: int, *, timeout: float = 5.0) -> None:
    """Assert that a process signalled by cleanup leaves the process table.

    Cleanup returns once its group probe finds no live member. On macOS that
    probe reports EPERM while a SIGKILLed descendant is still finishing its
    exit, and until then ``kill(pid, 0)`` still succeeds for that PID. A single
    immediate probe races that teardown, so allow bounded exit completion.

    This polls without yielding to the event loop, so cleanup cannot run in the
    meantime: a descendant it never signalled is still alive at the deadline.
    """
    deadline = time.monotonic() + timeout
    while True:
        try:
            os.kill(pid, 0)
        except ProcessLookupError:
            return
        assert time.monotonic() < deadline, "a descendant survived process-group cleanup"
        time.sleep(0.01)


class _ManualClockSelector(selectors.DefaultSelector):
    """Advance a manual clock by each requested wait instead of sleeping."""

    def __init__(self) -> None:
        super().__init__()
        self.now = 0.0

    def select(self, timeout: float | None = None) -> list[tuple[selectors.SelectorKey, int]]:
        if timeout is None:
            return super().select(None)
        self.now += max(timeout, 0.0)
        return super().select(0)


class _ManualClockEventLoop(asyncio.SelectorEventLoop):
    """Event loop whose timers elapse on a manual clock instead of the host's.

    Time advances only by the delay the loop asked to wait for, so a bound on
    timer-driven code counts the timeouts that code requested and not how long
    a loaded host descheduled the loop.
    """

    def __init__(self) -> None:
        self._manual_clock = _ManualClockSelector()
        super().__init__(self._manual_clock)

    def time(self) -> float:
        return self._manual_clock.now


def _provider(name: str, *, api_key: str = "k", base_url: str | None = None) -> ProviderConfig:
    return ProviderConfig(provider=name, model="m", api_key=api_key, base_url=base_url, litellm_model="m", region=None)


def test_streaming_log_redactor_is_chunk_partition_invariant() -> None:
    multiline_secret = "FIRST-HALF\nSECOND-HALF"
    shorter_secret = "overlap-secret"
    longer_secret = "overlap-secret-tail"
    collision_secret = "redacted"
    known_secret = "".join(("nvapi-", "Ab1Cd2Ef3Gh4Ij5Kl6Mn7Op8"))  # noqa: FLY002
    raw = (
        f"prefix {multiline_secret} {longer_secret} {shorter_secret} "
        f"{collision_secret} {known_secret} task-granularity suffix"
    )
    secrets = {multiline_secret, shorter_secret, longer_secret, collision_secret}

    baseline_redactor = StreamingLogRedactor(secrets)
    baseline = baseline_redactor.feed(raw) + baseline_redactor.finish()
    partitions = (
        [raw],
        list(raw),
        [raw[:1], raw[1:17], raw[17:43], raw[43:]],
        [raw[: len(raw) // 2], raw[len(raw) // 2 :]],
    )

    for chunks in partitions:
        redactor = StreamingLogRedactor(secrets)
        rendered = "".join(redactor.feed(chunk) for chunk in chunks) + redactor.finish()
        assert rendered == baseline

    assert "task-granularity" in baseline
    for secret in (*secrets, known_secret):
        assert secret not in baseline


def test_exact_stream_redactor_rejects_missing_compiled_prefix_matcher() -> None:
    redactor = StreamingSecretRedactor({"invariant-secret"})
    redactor._first_character_re = None

    with pytest.raises(RuntimeError, match="exact redactor invariant"):
        redactor.feed("ordinary output")


def test_known_pattern_redactor_rejects_unknown_candidate_kind() -> None:
    redactor = _StreamingKnownPatternRedactor()
    redactor._candidate_kind = "unexpected"
    redactor._candidate_prefix = "eyJ"
    redactor._candidate = ["eyJ"]

    with pytest.raises(RuntimeError, match="known-pattern redactor invariant"):
        redactor.feed("A")


@pytest.mark.parametrize("missing_stream", ("stdin", "stdout", "stderr"))
def test_local_stream_collector_rejects_missing_required_pipe(missing_stream: str) -> None:
    streams = {"stdin": object(), "stdout": object(), "stderr": object()}
    streams[missing_stream] = None
    process = SimpleNamespace(**streams)

    with pytest.raises(RuntimeError, match="local subprocess pipe invariant"):
        asyncio.run(
            SkillEvaluatorLocalEnvironment._collect_streamed_output(
                process,  # type: ignore[arg-type]
                b"",
                None,  # type: ignore[arg-type]
            )
        )


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@pytest.mark.parametrize("with_callback", [False, True])
def test_local_output_limit_fails_closed_and_reaps_process_group(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
    with_callback: bool,
) -> None:
    environment = _initialized_local_environment(tmp_path)
    environment._local_command_guardrail_reason = lambda *_args: ""  # type: ignore[method-assign]
    child_pid_path = environment._workspace / "output-limit-child-pid"
    overflow_value = "synthetic-output-overflow-secret"
    callback_chunks: list[str] = []
    processes: list[asyncio.subprocess.Process] = []
    create_subprocess_exec = asyncio.create_subprocess_exec

    async def capture_process(*args: object, **kwargs: object) -> asyncio.subprocess.Process:
        process = await create_subprocess_exec(*args, **kwargs)
        processes.append(process)
        return process

    async def on_output(text: str, _stream: str) -> None:
        callback_chunks.append(text)

    command = (
        "(sleep 30) & child=$!; "
        "printf '%s' \"$child\" > output-limit-child-pid; "
        "printf 12345; printf 67890 >&2; printf '%s' \"$OUTPUT_SECRET\"; wait"
    )

    async def exercise() -> None:
        await environment.start()
        callback_scope = environment.scoped_output_callback(on_output) if with_callback else contextlib.nullcontext()
        with callback_scope:
            await environment.exec(
                command,
                env={"OUTPUT_SECRET": overflow_value},
                # This case exercises output containment, not timeout racing.
                # Leave enough startup time for loaded CI hosts to emit the
                # deliberately over-budget bytes before the sleep descendant.
                timeout_sec=2.0,
            )

    monkeypatch.setattr(stream_redaction_module, "MAX_COMMAND_OUTPUT_BYTES", 8, raising=False)
    monkeypatch.setattr(asyncio, "create_subprocess_exec", capture_process)

    with pytest.raises(CommandOutputLimitError, match=r"Command output exceeded the 8-byte safety limit") as caught:
        asyncio.run(exercise())

    assert len(processes) == 1
    assert processes[0].returncode is not None
    assert child_pid_path.is_file()
    child_pid = int(child_pid_path.read_text(encoding="ascii"))
    _assert_process_exits(child_pid)
    assert len("".join(callback_chunks).encode()) <= 8
    assert overflow_value not in "".join(callback_chunks)
    assert overflow_value not in str(caught.value)


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_local_output_limit_redacts_detached_proxy_components_from_callback(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    environment = _initialized_local_environment(tmp_path)
    diagnostic = "proxy rejected local-limit-user with local-limit-password\n"
    callback_chunks: list[str] = []

    async def on_output(text: str, _stream: str) -> None:
        callback_chunks.append(text)

    async def exercise() -> None:
        await environment.start()
        with environment.scoped_output_callback(on_output):
            await environment.exec(
                f"printf %s {shlex.quote(diagnostic)}; sleep 0.05; printf overflow",
                env={
                    "HTTPS_PROXY": "https://local-limit-user:[email protected]:8443",
                },
            )

    monkeypatch.setattr(
        stream_redaction_module,
        "MAX_COMMAND_OUTPUT_BYTES",
        len(diagnostic.encode()),
        raising=False,
    )

    with pytest.raises(CommandOutputLimitError) as caught:
        asyncio.run(exercise())

    rendered = "".join(callback_chunks) + str(caught.value)
    assert "proxy rejected" in "".join(callback_chunks)
    assert "local-limit-user" not in rendered
    assert "local-limit-password" not in rendered


def test_streaming_log_redactor_preserves_nested_known_pattern_starts() -> None:
    jwt_secret = ".".join(("eyJ" + "A" * 20, "B" * 20, "C" * 20))
    raw = "ask-" + ("a" * 19 + "A1") + "--" + jwt_secret + "☃"
    expected = redact_secrets_in_log_line(raw)
    partitions = (
        [raw],
        list(raw),
        [raw[:5], raw[5:26], raw[26:51], raw[51:]],
    )

    for chunks in partitions:
        redactor = StreamingLogRedactor(())
        rendered = "".join(redactor.feed(chunk) for chunk in chunks) + redactor.finish()
        assert rendered == expected

    assert jwt_secret not in expected


@pytest.mark.parametrize(
    "raw",
    [
        " crsr_" + "0" * 16 + "sha256~" + "a" * 10 + "☃",
        "ordinary trailing xsk-abcdefgh",
        "ordinary trailing sk-short",
        "ordinary partial nvapi-",
        "ordinary partial crsr_0123",
        "ordinary partial eyJ" + "A" * 20,
        "_" + ".".join(("eyJ" + "A" * 20, "B" * 20, "C" * 20)),
        "é" + ".".join(("eyJ" + "A" * 20, "B" * 20, "C" * 20)),
    ],
)
def test_streaming_log_redactor_matches_batch_at_adjacent_patterns_and_eof(raw: str) -> None:
    expected = redact_secrets_in_log_line(raw)
    for chunks in ([raw], list(raw), [raw[: len(raw) // 2], raw[len(raw) // 2 :]]):
        redactor = StreamingLogRedactor(())
        rendered = "".join(redactor.feed(chunk) for chunk in chunks) + redactor.finish()
        assert rendered == expected


def test_streaming_log_redactor_emits_a_proven_long_key_before_its_terminator() -> None:
    redactor = StreamingLogRedactor(())

    emitted = redactor.feed("sk-" + "a" * 1_000_000)

    assert emitted == "sk-<redacted>"
    assert redactor.finish() == ""


@pytest.mark.parametrize(
    "prefix",
    [
        "eyJ",
        "eyJ" + "A" * 20 + ".",
    ],
)
def test_streaming_log_redactor_bounds_oversized_partial_jwt_candidates(prefix: str) -> None:
    redactor = StreamingLogRedactor(())
    raw = prefix + "A" * (512 * 1024)
    emitted: list[str] = []

    for offset in range(0, len(raw), 64 * 1024):
        emitted.append(redactor.feed(raw[offset : offset + 64 * 1024]))

    # Stage-zero/stage-one ambiguity must not retain attacker-sized output
    # until EOF. Once the conservative bound is reached, emit the marker and
    # discard the rest of that segment in bounded chunks.
    assert any(emitted)
    rendered = "".join(emitted) + redactor.finish()
    assert rendered == "jwt-<redacted>"
    assert len(rendered) < 256


@pytest.mark.parametrize(
    "raw",
    [
        "eyJ" + "A" * 256 + "." + "B" * 32 + "." + "C" * 32,
        "eyJ" + "A" * 20 + "." + "B" * 256 + "." + "C" * 32,
    ],
)
def test_oversized_partial_jwt_redaction_discards_later_segments(raw: str) -> None:
    redactor = StreamingLogRedactor(())

    rendered = redactor.feed(raw + " suffix") + redactor.finish()

    assert rendered == "jwt-<redacted> suffix"
    assert "B" * 32 not in rendered
    assert "C" * 32 not in rendered


@pytest.mark.parametrize(
    "known_secret",
    [
        "sk-abcdefgh",
        "nvapi-abcdefgh",
        "crsr_0123456789abcdef",
        "sha256~abcdefgh",
        ".".join(("eyJ" + "A" * 20, "B" * 20, "C" * 20)),
    ],
)
def test_exact_replacement_cannot_create_a_visible_known_secret(known_secret: str) -> None:
    exact_secret = "SECRET88"
    raw = exact_secret + known_secret
    for chunks in ([raw], list(raw), [raw[:7], raw[7:11], raw[11:]]):
        redactor = StreamingLogRedactor({exact_secret})
        rendered = "".join(redactor.feed(chunk) for chunk in chunks) + redactor.finish()
        assert exact_secret not in rendered
        assert known_secret not in rendered


def test_known_replacement_cannot_create_a_visible_exact_secret() -> None:
    exact_secret = "sk-<redacted>"
    raw = "sk-Ab1Cd2Ef3Gh4Ij5Kl6Mn7Op8"
    for chunks in ([raw], list(raw), [raw[:5], raw[5:19], raw[19:]]):
        redactor = StreamingLogRedactor({exact_secret})
        rendered = "".join(redactor.feed(chunk) for chunk in chunks) + redactor.finish()
        assert raw not in rendered
        assert exact_secret not in rendered


def test_local_is_a_registered_env_mode() -> None:
    assert ENV_MODE_LOCAL == "local"
    assert "local" in HARBOR_ENV_MODES


def test_registered_native_env_modes_are_supported_subset_of_pinned_harbor_release() -> None:
    from harbor.models.environment_type import EnvironmentType

    from skillevaluator.tier3_environments import harbor_environment_type

    harbor_modes = frozenset(environment.value for environment in EnvironmentType)
    harbor_types = frozenset(harbor_environment_type(mode) for mode in HARBOR_NATIVE_ENV_MODES)

    assert harbor_modes > harbor_types
    # Each SkillEvaluator mode maps to a pinned Harbor type; only aliases are not types themselves.
    assert HARBOR_NATIVE_ENV_MODES - harbor_modes == {"wandb"}
    # Harbor backends SkillEvaluator does not expose until it can project task bundles safely.
    assert harbor_modes - harbor_types == {
        "cua-cloud",
        "opensandbox",
        "hf-sandbox",
        "kernel",
        "podman",
        "prime",
        "sail",
        "sbx",
        "smol",
    }


def test_build_command_uses_unified_flags_for_local_imports() -> None:
    cmd = build_harbor_run_command(dataset_path="/tmp/ds", agent="opencode", job_name="j", env_mode="local")
    joined = " ".join(cmd)
    assert "--agent-import-path" not in cmd
    assert "--environment-import-path" not in cmd
    assert "-a" not in cmd
    assert cmd.count("--agent") == 1
    assert cmd[cmd.index("--agent") + 1] == LOCAL_AGENT_IMPORT_PATHS["opencode"]
    assert cmd.count("--env") == 1
    assert cmd[cmd.index("--env") + 1] == LOCAL_ENV_IMPORT_PATH
    assert "sandbox_mode=require" in joined
    assert "allow_net=true" in joined  # egress on by default for the live agent
    assert "runtime_agent=opencode" in joined
    assert "strict_reads=false" in joined


def test_build_command_wires_strict_read_policy(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setenv("SKILLEVALUATOR_LOCAL_STRICT_READS", "1")

    cmd = build_harbor_run_command(dataset_path="/tmp/ds", agent="opencode", job_name="j", env_mode="local")

    assert "strict_reads=true" in " ".join(cmd)


@pytest.mark.parametrize("agent", ["codex", "claude-code", "opencode"])
def test_build_command_docker_mode_uses_secure_import_path(agent: str) -> None:
    cmd = build_harbor_run_command(dataset_path="/tmp/ds", agent=agent, job_name="j", env_mode="docker")
    assert "--agent-import-path" not in cmd
    assert "--environment-import-path" not in cmd
    assert "-a" not in cmd
    assert cmd.count("--agent") == 1
    assert cmd[cmd.index("--agent") + 1] == DOCKER_AGENT_IMPORT_PATHS.get(agent, agent)
    assert cmd.count("--env") == 1
    assert cmd[cmd.index("--env") + 1] == SECURE_DOCKER_ENV_IMPORT_PATH


def test_docker_bridge_command_uses_unified_flags_for_custom_agent_and_secure_environment() -> None:
    import_path = "skillevaluator.tier3.harbor.local_agents:SkillEvaluatorNvidiaBuildCodex"

    cmd = build_harbor_run_command(
        dataset_path="/tmp/ds",
        agent="codex",
        job_name="nvidia-build-codex",
        env_mode="docker",
        agent_import_path=import_path,
    )

    assert "--agent-import-path" not in cmd
    assert "--environment-import-path" not in cmd
    assert "-a" not in cmd
    assert cmd.count("--agent") == 1
    assert cmd[cmd.index("--agent") + 1] == import_path
    assert cmd.count("--env") == 1
    assert cmd[cmd.index("--env") + 1] == SECURE_DOCKER_ENV_IMPORT_PATH


def test_nvidia_build_bridge_agents_are_not_local_mode_agents() -> None:
    assert NVIDIA_BUILD_AGENT_IMPORT_PATHS == {
        "codex": "skillevaluator.tier3.harbor.local_agents:SkillEvaluatorNvidiaBuildCodex",
        "claude-code": "skillevaluator.tier3.harbor.local_agents:SkillEvaluatorNvidiaBuildClaudeCode",
    }
    assert "codex" in LOCAL_AGENT_IMPORT_PATHS
    assert LOCAL_AGENT_IMPORT_PATHS["codex"] != NVIDIA_BUILD_AGENT_IMPORT_PATHS["codex"]


def test_harbor_unified_specs_import_non_abstract_skill_evaluator_classes(tmp_path: Path) -> None:
    import inspect

    from harbor.agents.factory import AgentFactory
    from harbor.cli.utils import resolve_environment_spec
    from harbor.environments.factory import EnvironmentFactory
    from harbor.models.task.config import EnvironmentConfig as TaskEnvironmentConfig
    from harbor.models.trial.config import AgentConfig, EnvironmentConfig
    from harbor.models.trial.paths import TrialPaths

    agent_specs = {
        "skillevaluator.tier3.harbor.local_agents:SkillEvaluatorLocalClaudeCode": ("SkillEvaluatorLocalClaudeCode"),
        "skillevaluator.tier3.harbor.local_agents:SkillEvaluatorLocalCodex": "SkillEvaluatorLocalCodex",
        "skillevaluator.tier3.harbor.local_agents:SkillEvaluatorLocalOpenCode": "SkillEvaluatorLocalOpenCode",
    }
    for import_path, expected_class_name in agent_specs.items():
        agent = AgentFactory.create_agent_from_config(
            AgentConfig(name=import_path, model_name="openai/gpt-4.1-mini"),
            logs_dir=tmp_path / "agent-logs",
        )

        assert agent.__class__.__name__ == expected_class_name
        assert not inspect.isabstract(agent.__class__)

    environment_dir = tmp_path / "environment"
    environment_dir.mkdir()
    (environment_dir / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8")
    environment_specs = {
        LOCAL_ENV_IMPORT_PATH: (
            "SkillEvaluatorLocalEnvironment",
            {
                "runtime_agent": "codex",
                "runtime_root": str(tmp_path / "runtime"),
                "sandbox_mode": "off",
            },
        ),
        SECURE_DOCKER_ENV_IMPORT_PATH: ("SkillEvaluatorSecureDockerEnvironment", {}),
    }
    for index, (import_path, (expected_class_name, kwargs)) in enumerate(environment_specs.items()):
        environment_type, resolved_import_path = resolve_environment_spec(import_path)
        assert environment_type is None
        assert resolved_import_path == import_path
        environment = EnvironmentFactory.create_environment_from_config(
            EnvironmentConfig(
                type=environment_type,
                import_path=resolved_import_path,
                kwargs=kwargs,
            ),
            environment_dir=environment_dir,
            environment_name="unified-import-smoke",
            session_id=f"unified-import-smoke-{index}",
            trial_paths=TrialPaths(tmp_path / f"trial-{index}"),
            task_env_config=TaskEnvironmentConfig(),
        )

        assert environment.__class__.__name__ == expected_class_name
        assert not inspect.isabstract(environment.__class__)


def test_local_claude_uses_managed_permissions_and_trial_temp_dir(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from harbor.agents.installed.claude_code import ClaudeCode

    from skillevaluator.tier3.harbor.local_agents import SkillEvaluatorLocalClaudeCode

    agent = object.__new__(SkillEvaluatorLocalClaudeCode)
    captured: dict[str, object] = {}

    async def raw_exec(
        _self: ClaudeCode,
        _environment: object,
        command: str,
        env: dict[str, str] | None = None,
        **_kwargs: object,
    ) -> SimpleNamespace:
        captured["command"] = command
        captured["env"] = dict(env or {})
        return SimpleNamespace(return_code=0)

    monkeypatch.setattr(ClaudeCode, "exec_as_agent", raw_exec)

    asyncio.run(
        agent.exec_as_agent(
            object(),
            "claude --permission-mode=bypassPermissions -- 'do not rewrite --permission-mode=bypassPermissions'",
            env={"CLAUDE_CODE_TMPDIR": "/private/tmp/unsafe"},
        )
    )

    command = str(captured["command"])
    launcher = command.partition(" -- ")[0]
    assert "--permission-mode=auto" in launcher
    assert "--permission-mode=bypassPermissions" not in launcher
    assert command.endswith("'do not rewrite --permission-mode=bypassPermissions'")
    assert command.startswith("mkdir -p /logs/agent/claude-tmp && ")
    assert captured["env"] == {"CLAUDE_CODE_TMPDIR": "/logs/agent/claude-tmp"}


def test_local_nvidia_build_codex_starts_authenticated_host_bridge_and_cleans_up(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    from harbor.agents.installed.codex import Codex

    from skillevaluator.tier3.harbor import local_agents

    agent_class = getattr(local_agents, "SkillEvaluatorLocalNvidiaBuildCodex", None)
    assert agent_class is not None
    agent = agent_class(logs_dir=tmp_path, model_name="nvidia/nemotron-3-nano-30b-a3b")
    agent.render_instruction = lambda instruction: instruction
    agent._resolve_auth_json_path = lambda: None
    agent._build_register_skills_command = lambda: None
    agent._build_register_mcp_servers_command = lambda: None
    agent.build_cli_flags = lambda: ""
    agent.logger = SimpleNamespace(debug=lambda *_args, **_kwargs: None)
    calls: list[tuple[str, dict[str, str]]] = []
    retained_logs: list[tuple[str, str]] = []
    origins: list[str] = []

    class Environment(_NoopScopedExecEnvironment):
        default_user = None

        async def upload_file(self, source: object, destination: object) -> None:
            retained_logs.append((str(destination), Path(source).read_text(encoding="utf-8")))

        async def exec_with_sensitive_env(
            self,
            command: str,
            env: dict[str, str] | None = None,
            **_kwargs: object,
        ) -> SimpleNamespace:
            calls.append((command, dict(env or {})))
            return SimpleNamespace(return_code=0, stdout="", stderr="")

    async def raw_exec(
        _self: Codex,
        _environment: object,
        command: str,
        env: dict[str, str] | None = None,
        **_kwargs: object,
    ) -> SimpleNamespace:
        calls.append((command, dict(env or {})))
        return SimpleNamespace(return_code=0)

    async def upstream_run(
        self: object,
        *,
        instruction: str,
        environment: object,
        context: object,
    ) -> None:
        _ = (instruction, context)
        origins.append(self._bridge_origin())
        effective_config = self._build_effective_config()
        await self._upload_effective_config(
            environment,
            effective_config,
            (self._REMOTE_CODEX_HOME / "config.toml").as_posix(),
        )
        await self.exec_as_agent(
            environment,
            command="codex exec --model nemotron-3-nano-30b-a3b -- test",
            env={"NVIDIA_API_KEY": "must-not-leak"},
        )

    monkeypatch.setattr(Codex, "exec_as_agent", raw_exec)
    monkeypatch.setattr(Codex, "run", upstream_run)
    monkeypatch.setenv("NVIDIA_API_KEY", "real-nvidia-key")

    asyncio.run(agent.run("test", Environment(), None))

    assert len(origins) == 1
    parsed = urlsplit(origins[0])
    assert parsed.hostname == "127.0.0.1"
    assert parsed.port is not None
    config_upload = next(content for destination, content in retained_logs if destination.endswith("/config.toml"))
    config = tomllib.loads(config_upload)
    assert config["model_provider"] == "openai_compatible"
    assert config["openai_base_url"] == f"{origins[0]}/v1"
    assert config["model_providers"]["openai_compatible"]["base_url"] == f"{origins[0]}/v1"
    assert "api.openai.com" not in config_upload
    assert "real-nvidia-key" not in config_upload
    client_command, client_env = next((command, env) for command, env in calls if "codex exec" in command)
    assert "env -u NVIDIA_API_KEY" in client_command
    assert "NVIDIA_API_KEY" not in client_env
    assert client_env["OPENAI_API_KEY"] not in {"real-nvidia-key", "nvidia-build-loopback"}
    assert any(destination.endswith("nvidia-build-bridge.log") for destination, _content in retained_logs)

    with socket.socket() as probe:
        probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        probe.bind(("127.0.0.1", parsed.port))


def test_local_nvidia_build_bridge_closes_if_cancelled_during_startup(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_agents

    agent = object.__new__(local_agents.SkillEvaluatorLocalNvidiaBuildClaudeCode)
    agent.model_name = "nvidia/nemotron-3.5-lightning-30b-a3b"
    agent._extra_env = {}
    worker_started = threading.Event()
    release_worker = threading.Event()
    closed = threading.Event()

    class Running:
        origin = "http://127.0.0.1:54321"
        client_token = "per-trial-capability"

        def close(self) -> None:
            closed.set()

    def delayed_start(**_kwargs: object) -> Running:
        worker_started.set()
        assert release_worker.wait(timeout=5)
        return Running()

    async def should_not_run(
        _self: object,
        *,
        instruction: str,
        environment: object,
        context: object,
    ) -> None:
        _ = (instruction, environment, context)
        pytest.fail("agent execution started after startup cancellation")

    class Environment:
        async def upload_file(self, _source: object, _destination: object) -> None:
            return None

    monkeypatch.setattr(local_agents, "start_in_process_bridge", delayed_start)
    monkeypatch.setattr(local_agents.SkillEvaluatorLocalClaudeCode, "run", should_not_run)
    monkeypatch.setenv("NVIDIA_API_KEY", "real-nvidia-key")

    async def exercise() -> None:
        task = asyncio.create_task(agent.run("test", Environment(), None))
        assert await asyncio.to_thread(worker_started.wait, 5)
        task.cancel()
        await asyncio.sleep(0)
        task.cancel()
        await asyncio.sleep(0)
        release_worker.set()
        with pytest.raises(asyncio.CancelledError):
            await task

    asyncio.run(exercise())

    assert closed.is_set()
    assert getattr(agent, "_nvidia_build_local_temp_dir", None) is None


def test_local_nvidia_build_bridge_cleanup_preserves_cancellation(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_agents

    close_started = threading.Event()
    release_close = threading.Event()
    close_finished = threading.Event()

    class Running:
        def close(self) -> None:
            close_started.set()
            assert release_close.wait(timeout=5)
            close_finished.set()

    async def exercise() -> None:
        task = asyncio.create_task(local_agents._close_running_bridge(Running()))
        assert await asyncio.to_thread(close_started.wait, 5)
        task.cancel()
        await asyncio.sleep(0)
        release_close.set()
        with pytest.raises(asyncio.CancelledError):
            await task

    asyncio.run(exercise())

    assert close_finished.is_set()


def test_start_in_process_bridge_waits_for_readiness_worker_to_finish(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
    from skillevaluator.tier3.harbor import nvidia_build_bridge

    release_started = threading.Event()
    release_allowed = threading.Event()
    release_lock = threading.Lock()
    original_release_request = nvidia_build_bridge._BridgeHTTPServer.release_request
    first_release = True

    def delayed_first_release(server: object, request: object) -> None:
        nonlocal first_release
        with release_lock:
            delay_release = first_release
            first_release = False
        if delay_release:
            release_started.set()
            release_timer = threading.Timer(1.0, release_allowed.set)
            release_timer.daemon = True
            release_timer.start()
            assert release_allowed.wait(timeout=2)
        original_release_request(server, request)

    monkeypatch.setattr(nvidia_build_bridge._BridgeHTTPServer, "release_request", delayed_first_release)

    running = nvidia_build_bridge.start_in_process_bridge(
        api_key="test-readiness-worker-key",
        build_base_url="http://127.0.0.1:8080/v1",
        log_path=tmp_path / "nvidia-build-bridge.log",
        request_transport=lambda _endpoint, _body: b"{}",
    )
    try:
        assert release_started.wait(timeout=1)
        assert running._server.active_workers == 0
    finally:
        release_allowed.set()
        running.close()


def test_local_nvidia_build_bridge_repeated_cancellation_releases_slow_header_state(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
    from skillevaluator.tier3.harbor import local_agents, nvidia_build_bridge

    temp_dir = tempfile.TemporaryDirectory(prefix="bridge-repeated-cancel-")
    temp_path = Path(temp_dir.name)

    def transport(_endpoint: str, _body: bytes) -> bytes:
        return json.dumps(
            {
                "id": "chatcmpl-test",
                "model": "nvidia/model",
                "choices": [{"message": {"role": "assistant", "content": "ok"}}],
            }
        ).encode("utf-8")

    running = nvidia_build_bridge.start_in_process_bridge(
        api_key="test-repeated-cancel-key",
        build_base_url="http://127.0.0.1:8080/v1",
        log_path=temp_path / "nvidia-build-bridge.log",
        request_transport=transport,
    )
    port = int(urlsplit(running.origin).port or 0)
    monkeypatch.setattr(nvidia_build_bridge, "REQUEST_HEADER_TIMEOUT_SECONDS", 60.0)
    monkeypatch.setattr(nvidia_build_bridge, "BACKEND_CONNECT_TIMEOUT_SECONDS", 0.05)
    monkeypatch.setattr(nvidia_build_bridge, "IN_PROCESS_START_TIMEOUT_SECONDS", 0.05)
    client = socket.create_connection(("127.0.0.1", port), timeout=2)
    stop_sending = threading.Event()

    def drip_headers() -> None:
        while not stop_sending.is_set():
            try:
                client.sendall(b"P")
            except OSError:
                return
            time.sleep(0.015)

    sender = threading.Thread(target=drip_headers)
    sender.start()
    deadline = time.monotonic() + 1
    while running._server.active_workers == 0 and time.monotonic() < deadline:
        time.sleep(0.005)
    assert running._server.active_workers == 1

    close_started = threading.Event()
    release_close = threading.Event()
    original_close = running.close

    def delayed_close() -> None:
        close_started.set()
        assert release_close.wait(timeout=5)
        original_close()

    monkeypatch.setattr(running, "close", delayed_close)
    agent = object.__new__(local_agents.SkillEvaluatorLocalNvidiaBuildCodex)
    agent._nvidia_build_running_bridge = running
    agent._nvidia_build_local_log_path = temp_path / "nvidia-build-bridge.log"
    agent._nvidia_build_local_temp_dir = temp_dir
    agent._nvidia_build_bridge_client_token = running.client_token
    agent._nvidia_build_bridge_origin = running.origin
    agent._nvidia_build_bridge_started = True

    class Environment:
        async def upload_file(self, _source: object, _destination: object) -> None:
            pytest.fail("cancelled cleanup must not upload after cancellation")

    async def exercise() -> None:
        task = asyncio.create_task(agent._cleanup_bridge(Environment()))
        assert await asyncio.to_thread(close_started.wait, 5)
        task.cancel()
        await asyncio.sleep(0)
        task.cancel()
        await asyncio.sleep(0)
        release_close.set()
        with pytest.raises(asyncio.CancelledError):
            await task

    try:
        asyncio.run(exercise())
    finally:
        release_close.set()
        stop_sending.set()
        client.close()
        sender.join(timeout=1)
        if not running._closed:
            running.close()

    assert not sender.is_alive()
    assert running._closed is True
    assert agent._nvidia_build_running_bridge is None
    assert agent._nvidia_build_local_log_path is None
    assert agent._nvidia_build_local_temp_dir is None
    assert agent._nvidia_build_bridge_client_token is None
    assert agent._nvidia_build_bridge_origin is None
    assert agent._nvidia_build_bridge_started is False
    assert not temp_path.exists()


def test_nvidia_build_codex_bridge_isolated_from_client_and_cleans_up(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    from harbor.agents.installed.codex import Codex

    agent = SkillEvaluatorNvidiaBuildCodex(
        logs_dir=tmp_path,
        model_name="nvidia/meta/llama-3.1-8b-instruct",
    )
    calls: list[tuple[str, dict[str, str]]] = []
    root_calls: list[tuple[str, dict[str, str]]] = []
    uploads: list[tuple[str, str, int]] = []

    class Environment(_NoopScopedExecEnvironment):
        default_user = None

        async def upload_file(self, source: object, destination: object) -> None:
            source_path = Path(source)
            uploads.append(
                (str(destination), source_path.read_text(encoding="utf-8"), source_path.stat().st_mode & 0o777)
            )

        async def exec_with_sensitive_env(
            self,
            command: str,
            env: dict[str, str] | None = None,
            user: str | int | None = None,
            **_kwargs: object,
        ) -> SimpleNamespace:
            destination = root_calls if user == "root" else calls
            destination.append((command, dict(env or {})))
            return SimpleNamespace(return_code=0, stdout="", stderr="")

    async def raw_exec(
        _self: Codex,
        _environment: object,
        command: str,
        env: dict[str, str] | None = None,
        **_kwargs: object,
    ) -> SimpleNamespace:
        calls.append((command, dict(env or {})))
        stdout = "CODEX_RUNTIME_ID 1000 1000\n" if "CODEX_RUNTIME_ID" in command else ""
        return SimpleNamespace(return_code=0, stdout=stdout, stderr="")

    async def root_exec(
        _self: Codex,
        _environment: object,
        command: str,
        env: dict[str, str] | None = None,
        **_kwargs: object,
    ) -> SimpleNamespace:
        root_calls.append((command, dict(env or {})))
        stdout = "http://127.0.0.1:43123\n" if "--check-ready-file" in command else ""
        return SimpleNamespace(return_code=0, stdout=stdout)

    monkeypatch.setattr(Codex, "exec_as_agent", raw_exec)
    monkeypatch.setattr(Codex, "exec_as_root", root_exec)
    monkeypatch.setenv("NVIDIA_API_KEY", "nvidia-secret")
    agent._extra_env = {}
    agent.render_instruction = lambda instruction: instruction
    agent._resolve_auth_json_path = lambda: None
    agent._build_register_skills_command = lambda: None
    agent._build_register_mcp_servers_command = lambda: None
    agent.build_cli_flags = lambda: ""
    agent.logger = SimpleNamespace(debug=lambda *_args, **_kwargs: None)

    asyncio.run(agent.run("test", Environment(), None))

    assert len(uploads) == 2
    assert uploads[0][0].endswith("nvidia-build-bridge.py")
    secret_handoff = next(
        (command, env) for command, env in root_calls if "SKILLEVALUATOR_NVIDIA_BUILD_BRIDGE_API_KEY" in env
    )
    assert secret_handoff[1]["SKILLEVALUATOR_NVIDIA_BUILD_BRIDGE_API_KEY"] == "nvidia-secret"
    client_token = secret_handoff[1]["SKILLEVALUATOR_NVIDIA_BUILD_BRIDGE_CLIENT_TOKEN"]
    assert client_token not in {"", "nvidia-secret", "nvidia-build-loopback"}
    assert len(client_token) >= 32
    assert "nvidia-secret" not in secret_handoff[0]
    assert client_token not in secret_handoff[0]
    assert ".key" in secret_handoff[0]
    assert ".token" in secret_handoff[0]
    bridge_start = next(
        (command, env) for command, env in root_calls if "nvidia-build-bridge.py" in command and "&" in command
    )
    assert bridge_start[1] == {}
    assert "--api-key-file" in bridge_start[0]
    assert "--client-token-file" in bridge_start[0]
    assert "chown 0:0" in bridge_start[0]
    assert bridge_start[0].index("chown 0:0") < bridge_start[0].index("chmod 600")
    assert "--allowed-model nvidia/meta/llama-3.1-8b-instruct" in bridge_start[0]
    assert "--max-requests" in bridge_start[0]
    assert "skillevaluator-nvidia-build-" in bridge_start[0]
    assert "nvidia-secret" not in bridge_start[0]
    assert client_token not in bridge_start[0]
    assert "--port 0" in bridge_start[0]
    assert "--ready-file" in bridge_start[0]
    assert "18080" not in bridge_start[0]
    health_command = next(command for command, _env in root_calls if "--check-ready-file" in command)
    assert "kill -0" in health_command
    assert "/healthz" not in health_command
    assert all("NVIDIA_API_KEY" not in env for _, env in [*calls, *root_calls])
    config_upload = next(content for destination, content, _mode in uploads if destination.endswith("/config.toml"))
    config = tomllib.loads(config_upload)
    assert config["model_provider"] == "openai_compatible"
    assert config["openai_base_url"] == "http://127.0.0.1:43123/v1"
    assert config["model_providers"]["openai_compatible"] == {
        "name": "OpenAI-compatible provider",
        "base_url": "http://127.0.0.1:43123/v1",
        "env_key": "OPENAI_API_KEY",
        "wire_api": "responses",
    }
    assert all("openai_base_url" not in command for command, _ in calls)
    assert "nvidia-secret" not in config_upload
    client_command, client_env = next((command, env) for command, env in calls if "codex exec" in command)
    assert "NVIDIA_API_KEY" not in client_env
    assert "OPENAI_BASE_URL" not in client_env
    assert client_env["OPENAI_API_KEY"] != "nvidia-secret"
    assert "env -u NVIDIA_API_KEY" in client_command
    assert "--model nvidia/meta/llama-3.1-8b-instruct" in client_command
    cleanup = next(command for command, _ in root_calls if 'kill "$(cat' in command)
    assert "skillevaluator-nvidia-build-" in cleanup
    assert "nvidia-build-bridge.ready" in cleanup
    assert "nvidia-build-bridge.py" in cleanup
    assert ".key" in cleanup
    assert ".token" in cleanup


def test_nvidia_build_container_bridge_streams_secrets_without_host_temp_files(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from harbor.agents.installed.codex import Codex

    from skillevaluator.tier3.harbor import local_agents

    agent = object.__new__(SkillEvaluatorNvidiaBuildCodex)
    agent.model_name = "nvidia/model"
    uploads: list[str] = []
    root_calls: list[tuple[str, dict[str, str]]] = []

    class Environment:
        async def upload_file(self, _source: object, destination: object) -> None:
            uploads.append(str(destination))

        async def exec_with_sensitive_env(
            self,
            command: str,
            env: dict[str, str] | None = None,
            **_kwargs: object,
        ) -> SimpleNamespace:
            root_calls.append((command, dict(env or {})))
            return SimpleNamespace(return_code=0, stdout="")

    def reject_host_secret_file(*_args, **_kwargs):
        raise AssertionError("bridge secrets must not be materialized in host temporary files")

    async def root_exec(
        _self: Codex,
        _environment: object,
        command: str,
        env: dict[str, str] | None = None,
        **_kwargs: object,
    ) -> SimpleNamespace:
        root_calls.append((command, dict(env or {})))
        stdout = "http://127.0.0.1:43123\n" if "--check-ready-file" in command else ""
        return SimpleNamespace(return_code=0, stdout=stdout)

    monkeypatch.setattr(local_agents.tempfile, "mkstemp", reject_host_secret_file)
    monkeypatch.setattr(Codex, "exec_as_root", root_exec)
    monkeypatch.setenv("NVIDIA_API_KEY", "nvidia-secret")

    asyncio.run(agent._start_bridge(Environment()))
    asyncio.run(agent._cleanup_bridge(Environment()))

    assert len(uploads) == 1
    assert uploads[0].endswith("nvidia-build-bridge.py")
    secret_handoff = next((command, env) for command, env in root_calls if ".key" in command and ".token" in command)
    assert secret_handoff[1]["SKILLEVALUATOR_NVIDIA_BUILD_BRIDGE_API_KEY"] == "nvidia-secret"
    client_token = secret_handoff[1]["SKILLEVALUATOR_NVIDIA_BUILD_BRIDGE_CLIENT_TOKEN"]
    assert len(client_token) >= 32
    assert "nvidia-secret" not in secret_handoff[0]
    assert client_token not in secret_handoff[0]
    assert "umask 077" in secret_handoff[0]


@pytest.mark.parametrize(
    "cancel_stage",
    ["bridge-script-upload", "secret-handoff", "startup", "health-check"],
)
def test_nvidia_build_container_bridge_cancellation_cleans_all_private_state_uninterruptibly(
    monkeypatch: pytest.MonkeyPatch,
    cancel_stage: str,
) -> None:
    from harbor.agents.installed.codex import Codex

    agent = object.__new__(SkillEvaluatorNvidiaBuildCodex)
    agent.model_name = "nvidia/model"
    stage_reached = asyncio.Event()
    cleanup_started = asyncio.Event()
    release_cleanup = asyncio.Event()
    cleanup_commands: list[str] = []

    class Environment:
        async def upload_file(self, _source: object, destination: object) -> None:
            destination_text = str(destination)
            stage = (
                "bridge-script-upload"
                if destination_text.endswith("nvidia-build-bridge.py")
                else "api-key-upload"
                if destination_text.endswith(".key")
                else "client-token-upload"
                if destination_text.endswith(".token")
                else "unknown-upload"
            )
            if stage == cancel_stage:
                stage_reached.set()
                await asyncio.Future()

        async def exec_with_sensitive_env(
            self,
            command: str,
            env: dict[str, str] | None = None,
            **_kwargs: object,
        ) -> SimpleNamespace:
            del command
            if env and "SKILLEVALUATOR_NVIDIA_BUILD_BRIDGE_API_KEY" in env and cancel_stage == "secret-handoff":
                stage_reached.set()
                await asyncio.Future()
            return SimpleNamespace(return_code=0, stdout="", stderr="")

    async def root_exec(
        _self: Codex,
        _environment: object,
        command: str,
        **kwargs: object,
    ) -> SimpleNamespace:
        if 'kill "$(cat' in command:
            cleanup_commands.append(command)
            cleanup_started.set()
            await release_cleanup.wait()
            return SimpleNamespace(return_code=0, stdout="")
        environment = kwargs.get("env")
        stage = (
            "secret-handoff"
            if isinstance(environment, dict) and "SKILLEVALUATOR_NVIDIA_BUILD_BRIDGE_API_KEY" in environment
            else "health-check"
            if "--check-ready-file" in command
            else "startup"
            if "nvidia-build-bridge.py" in command and "&" in command
            else "health-check"
        )
        if stage == cancel_stage:
            stage_reached.set()
            await asyncio.Future()
        stdout = "http://127.0.0.1:43123\n" if stage == "health-check" else ""
        return SimpleNamespace(return_code=0, stdout=stdout)

    monkeypatch.setattr(Codex, "exec_as_root", root_exec)
    monkeypatch.setenv("NVIDIA_API_KEY", "nvidia-secret")

    async def exercise() -> None:
        nonlocal stage_reached, cleanup_started, release_cleanup
        stage_reached = asyncio.Event()
        cleanup_started = asyncio.Event()
        release_cleanup = asyncio.Event()
        task = asyncio.create_task(agent._start_bridge(Environment()))
        try:
            await asyncio.wait_for(stage_reached.wait(), timeout=1)
            task.cancel()
            await asyncio.wait_for(cleanup_started.wait(), timeout=1)
            task.cancel()
            release_cleanup.set()
            with pytest.raises(asyncio.CancelledError):
                await task
        finally:
            release_cleanup.set()
            if not task.done():
                task.cancel()
                with contextlib.suppress(asyncio.CancelledError):
                    await task

    asyncio.run(exercise())

    assert len(cleanup_commands) == 1
    assert ".key" in cleanup_commands[0]
    assert ".token" in cleanup_commands[0]
    assert agent._nvidia_build_bridge_started is False
    assert agent._nvidia_build_bridge_key_file is None
    assert agent._nvidia_build_bridge_client_token_file is None
    assert agent._nvidia_build_bridge_client_token is None


def test_nvidia_build_bridge_prefers_file_backed_host_key_over_subprocess_sentinel(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    from harbor.agents.installed.codex import Codex

    agent = object.__new__(SkillEvaluatorNvidiaBuildCodex)
    agent.model_name = "nvidia/model"
    agent._nvidia_build_bridge_started = False
    agent._nvidia_build_bridge_key_file = None
    root_calls: list[tuple[str, dict[str, str]]] = []
    uploads: list[str] = []
    host_key_file = tmp_path / "nvidia-build-host-key"
    host_key_file.write_text("real-nvidia-secret", encoding="utf-8")

    class Environment:
        async def upload_file(self, source: object, destination: object) -> None:
            del source
            uploads.append(str(destination))

        async def exec_with_sensitive_env(
            self,
            command: str,
            env: dict[str, str] | None = None,
            **_kwargs: object,
        ) -> SimpleNamespace:
            root_calls.append((command, dict(env or {})))
            return SimpleNamespace(return_code=0, stdout="", stderr="")

    async def root_exec(
        _self: Codex,
        _environment: object,
        command: str,
        env: dict[str, str] | None = None,
        **_kwargs: object,
    ) -> SimpleNamespace:
        root_calls.append((command, dict(env or {})))
        stdout = "http://127.0.0.1:43123\n" if "--check-ready-file" in command else ""
        return SimpleNamespace(return_code=0, stdout=stdout)

    monkeypatch.setattr(Codex, "exec_as_root", root_exec)
    monkeypatch.setenv("NVIDIA_API_KEY", "skillevaluator-file-backed-nvidia-key")
    monkeypatch.setenv("SKILLEVALUATOR_NVIDIA_API_KEY_FILE", str(host_key_file))

    asyncio.run(agent._start_bridge(Environment()))
    asyncio.run(agent._cleanup_bridge(Environment()))

    assert len(uploads) == 1
    assert uploads[0].endswith("nvidia-build-bridge.py")
    secret_handoff = next(
        (command, env) for command, env in root_calls if "SKILLEVALUATOR_NVIDIA_BUILD_BRIDGE_API_KEY" in env
    )
    assert secret_handoff[1]["SKILLEVALUATOR_NVIDIA_BUILD_BRIDGE_API_KEY"] == "real-nvidia-secret"
    client_token = secret_handoff[1]["SKILLEVALUATOR_NVIDIA_BUILD_BRIDGE_CLIENT_TOKEN"]
    assert len(client_token) >= 32
    assert host_key_file.exists()
    assert all("real-nvidia-secret" not in command for command, _env in root_calls)
    assert all("skillevaluator-file-backed-nvidia-key" not in command for command, _env in root_calls)
    assert client_token not in secret_handoff[0]


def test_nvidia_build_bridge_rejects_file_backed_sentinel_without_host_key_file(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    agent = object.__new__(SkillEvaluatorNvidiaBuildCodex)

    class Environment:
        async def exec_with_sensitive_env(self, **_kwargs: object) -> SimpleNamespace:
            raise AssertionError("credential validation must fail before execution")

    monkeypatch.setenv("NVIDIA_API_KEY", "skillevaluator-file-backed-nvidia-key")
    monkeypatch.delenv("SKILLEVALUATOR_NVIDIA_API_KEY_FILE", raising=False)

    with pytest.raises(RuntimeError, match="SKILLEVALUATOR_NVIDIA_API_KEY_FILE"):
        asyncio.run(agent._start_bridge(Environment()))


def test_nvidia_build_bridge_rejects_unsafe_environment_before_reading_or_transferring_key(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_agents

    agent = object.__new__(SkillEvaluatorNvidiaBuildCodex)
    agent.model_name = "nvidia/model"
    uploads: list[str] = []
    executions: list[tuple[str, dict[str, str]]] = []
    key_reads: list[bool] = []

    class UnsafeEnvironment:
        async def upload_file(self, _source: object, destination: object) -> None:
            uploads.append(str(destination))

        async def exec(
            self,
            command: str,
            env: dict[str, str] | None = None,
            **_kwargs: object,
        ) -> SimpleNamespace:
            executions.append((command, dict(env or {})))
            return SimpleNamespace(return_code=0, stdout="", stderr="")

    def reject_key_read() -> str:
        key_reads.append(True)
        raise AssertionError("unsupported environments must be rejected before consuming stdin")

    monkeypatch.setattr(local_agents, "read_nvidia_build_key_from_stdin", reject_key_read)
    monkeypatch.setenv("NVIDIA_API_KEY", local_agents.NVIDIA_BUILD_STDIN_SENTINEL)

    with pytest.raises(RuntimeError, match="protected sensitive-value transport"):
        asyncio.run(agent._start_bridge(UnsafeEnvironment()))

    assert key_reads == []
    assert uploads == []
    assert executions == []


def test_nvidia_build_bridge_health_failure_cleans_up_before_raising(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from harbor.agents.installed.codex import Codex

    agent = object.__new__(SkillEvaluatorNvidiaBuildCodex)
    agent.model_name = "nvidia/model"
    commands: list[str] = []

    class Environment:
        async def upload_file(self, _source: object, _destination: object) -> None:
            return None

        async def exec_with_sensitive_env(
            self,
            command: str,
            env: dict[str, str] | None = None,
            **_kwargs: object,
        ) -> SimpleNamespace:
            commands.append(command)
            assert env and "SKILLEVALUATOR_NVIDIA_BUILD_BRIDGE_API_KEY" in env
            return SimpleNamespace(return_code=0, stdout="", stderr="")

    async def raw_exec(
        _self: Codex,
        _environment: object,
        command: str,
        **_kwargs: object,
    ) -> SimpleNamespace:
        commands.append(command)
        is_health = "/healthz" in command or "--check-ready-file" in command
        return SimpleNamespace(return_code=1 if is_health else 0, stdout="")

    monkeypatch.setattr(Codex, "exec_as_agent", raw_exec)
    monkeypatch.setattr(Codex, "exec_as_root", raw_exec)
    monkeypatch.setenv("NVIDIA_API_KEY", "nvidia-secret")

    with pytest.raises(RuntimeError, match="health check"):
        asyncio.run(agent.run("test", Environment(), None))

    assert any("--check-ready-file" in command for command in commands)
    assert any('kill "$(cat' in command for command in commands)


def test_nvidia_build_bridge_start_failure_removes_uploaded_key(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from harbor.agents.installed.codex import Codex

    agent = object.__new__(SkillEvaluatorNvidiaBuildCodex)
    agent.model_name = "nvidia/model"
    commands: list[str] = []

    class Environment:
        async def upload_file(self, _source: object, _destination: object) -> None:
            return None

        async def exec_with_sensitive_env(
            self,
            command: str,
            env: dict[str, str] | None = None,
            **_kwargs: object,
        ) -> SimpleNamespace:
            commands.append(command)
            assert env and "SKILLEVALUATOR_NVIDIA_BUILD_BRIDGE_API_KEY" in env
            return SimpleNamespace(return_code=0, stdout="", stderr="")

    async def raw_exec(
        _self: Codex,
        _environment: object,
        command: str,
        **_kwargs: object,
    ) -> SimpleNamespace:
        commands.append(command)
        is_start = "nvidia-build-bridge.py" in command and "&" in command
        return SimpleNamespace(return_code=1 if is_start else 0)

    monkeypatch.setattr(Codex, "exec_as_agent", raw_exec)
    monkeypatch.setattr(Codex, "exec_as_root", raw_exec)
    monkeypatch.setenv("NVIDIA_API_KEY", "nvidia-secret")

    with pytest.raises(RuntimeError, match="startup"):
        asyncio.run(agent.run("test", Environment(), None))

    cleanup = next(command for command in commands if 'kill "$(cat' in command)
    assert "skillevaluator-nvidia-build-" in cleanup
    assert "nvidia-build-bridge.ready" in cleanup


def test_nvidia_build_bridge_start_exception_removes_uploaded_key(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from harbor.agents.installed.codex import Codex

    agent = object.__new__(SkillEvaluatorNvidiaBuildCodex)
    agent.model_name = "nvidia/model"
    commands: list[str] = []

    class Environment:
        async def upload_file(self, _source: object, _destination: object) -> None:
            return None

        async def exec_with_sensitive_env(
            self,
            command: str,
            env: dict[str, str] | None = None,
            **_kwargs: object,
        ) -> SimpleNamespace:
            commands.append(command)
            assert env and "SKILLEVALUATOR_NVIDIA_BUILD_BRIDGE_API_KEY" in env
            return SimpleNamespace(return_code=0, stdout="", stderr="")

    async def raw_exec(
        _self: Codex,
        _environment: object,
        command: str,
        **_kwargs: object,
    ) -> SimpleNamespace:
        commands.append(command)
        if "nvidia-build-bridge.py" in command and "&" in command:
            raise RuntimeError("docker exec failed")
        return SimpleNamespace(return_code=0)

    monkeypatch.setattr(Codex, "exec_as_agent", raw_exec)
    monkeypatch.setattr(Codex, "exec_as_root", raw_exec)
    monkeypatch.setenv("NVIDIA_API_KEY", "nvidia-secret")

    with pytest.raises(RuntimeError, match="docker exec failed"):
        asyncio.run(agent.run("test", Environment(), None))

    cleanup = next(command for command in commands if 'kill "$(cat' in command)
    assert "skillevaluator-nvidia-build-" in cleanup
    assert "nvidia-build-bridge.ready" in cleanup


def test_nvidia_build_bridge_wraps_compound_codex_shell_commands_before_unsetting_key(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from harbor.agents.installed.codex import Codex

    agent = object.__new__(SkillEvaluatorNvidiaBuildCodex)
    agent.model_name = "nvidia/model"
    agent._nvidia_build_bridge_origin = "http://127.0.0.1:43123"
    agent._nvidia_build_bridge_client_env = {"OPENAI_API_KEY": "bridge-client-token-secret"}
    captured: list[tuple[str, dict[str, str]]] = []
    scoped: list[dict[str, str]] = []
    simple_command = "codex exec --model model -- test"
    compound_command = 'if [ -d "$CODEX_HOME/sessions" ]; then cp -R "$CODEX_HOME/sessions" /logs/agent/sessions; fi'

    class Environment:
        @contextlib.contextmanager
        def scoped_exec_env(self, values: dict[str, str]):
            scoped.append(dict(values))
            yield

    async def raw_exec(
        _self: Codex,
        _environment: object,
        command: str,
        env: dict[str, str] | None = None,
        **_kwargs: object,
    ) -> SimpleNamespace:
        captured.append((command, dict(env or {})))
        return SimpleNamespace(return_code=0, stdout="", stderr="")

    monkeypatch.setattr(Codex, "exec_as_agent", raw_exec)

    environment = Environment()
    for original_command in (simple_command, compound_command):
        asyncio.run(
            agent.exec_as_agent(
                environment,
                command=original_command,
                env={"NVIDIA_API_KEY": "must-not-leak"},
            )
        )

    unset_prefix = "env -u NVIDIA_API_KEY -u OPENAI_BASE_URL -u OPENAI_API_BASE bash -o pipefail -c"
    assert [command for command, _ in captured] == [
        f"{unset_prefix} {shlex.quote('codex exec --model nvidia/model -- test')}",
        f"{unset_prefix} {shlex.quote(compound_command)}",
    ]
    assert all("NVIDIA_API_KEY" not in env for _, env in captured)
    assert all(env["OPENAI_API_KEY"] == "bridge-client-token-secret" for _, env in captured)
    assert scoped == [
        {"OPENAI_API_KEY": "bridge-client-token-secret"},
        {"OPENAI_API_KEY": "bridge-client-token-secret"},
    ]


def test_nvidia_build_claude_bridge_configures_origin_and_full_model(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from harbor.agents.installed.claude_code import ClaudeCode

    agent = object.__new__(SkillEvaluatorNvidiaBuildClaudeCode)
    agent.model_name = "nvidia/meta/llama-3.1-8b-instruct"
    calls: list[tuple[str, dict[str, str]]] = []

    class Environment(_NoopScopedExecEnvironment):
        async def upload_file(self, _source: object, _destination: object) -> None:
            return None

        async def exec_with_sensitive_env(
            self,
            command: str,
            env: dict[str, str] | None = None,
            **_kwargs: object,
        ) -> SimpleNamespace:
            calls.append((command, dict(env or {})))
            return SimpleNamespace(return_code=0, stdout="", stderr="")

    async def raw_exec(
        _self: ClaudeCode,
        _environment: object,
        command: str,
        env: dict[str, str] | None = None,
        **_kwargs: object,
    ) -> SimpleNamespace:
        calls.append((command, dict(env or {})))
        stdout = "http://127.0.0.1:43123\n" if "--check-ready-file" in command else ""
        return SimpleNamespace(return_code=0, stdout=stdout)

    async def parent_run(
        self: ClaudeCode,
        *,
        instruction: str,
        environment: object,
        context: object,
    ) -> None:
        _ = (instruction, context)
        await self.exec_as_agent(environment, command="claude --print -- test", env={})

    monkeypatch.setattr(ClaudeCode, "exec_as_agent", raw_exec)
    monkeypatch.setattr(ClaudeCode, "exec_as_root", raw_exec)
    monkeypatch.setattr(ClaudeCode, "run", parent_run)
    monkeypatch.setenv("NVIDIA_API_KEY", "nvidia-secret")

    asyncio.run(agent.run("test", Environment(), None))

    client_command, client_env = next((command, env) for command, env in calls if "claude --print" in command)
    assert "env -u NVIDIA_API_KEY" in client_command
    assert "NVIDIA_API_KEY" not in client_env
    # This is configuration-only: a live Docker smoke test is required to
    # verify the installed Claude CLI's actual request construction.
    assert client_env["ANTHROPIC_BASE_URL"] == "http://127.0.0.1:43123"
    assert client_env["ANTHROPIC_API_KEY"] != "nvidia-secret"
    assert client_env["ANTHROPIC_MODEL"] == "nvidia/meta/llama-3.1-8b-instruct"


@pytest.mark.live
@pytest.mark.skip(
    reason=(
        "Manual Docker E2E only: requires NVIDIA_API_KEY, Docker, and the installed Claude Code CLI; "
        "do not run in the unit suite."
    )
)
def test_nvidia_build_claude_bridge_live_smoke() -> None:
    """Manual scope: prove a Docker Claude CLI request reaches bridge /v1/messages."""
    pytest.fail("run the documented manual Docker E2E smoke with a real NVIDIA Build credential")


def test_local_agent_credentials_map_provider_to_agent_env() -> None:
    nv = _local_agent_credentials(
        _provider("nv_build", api_key="nvapi-x", base_url="https://integrate.api.nvidia.com/v1")
    )
    assert nv == {"OPENAI_API_KEY": "nvapi-x", "OPENAI_BASE_URL": "https://integrate.api.nvidia.com/v1"}
    anthropic = _local_agent_credentials(_provider("anthropic", api_key="sk-ant"))
    assert anthropic == {"ANTHROPIC_API_KEY": "sk-ant"}
    openai = _local_agent_credentials(_provider("openai", api_key="sk-o", base_url="https://api.openai.com/v1"))
    assert openai == {"OPENAI_API_KEY": "sk-o", "OPENAI_BASE_URL": "https://api.openai.com/v1"}


def test_local_subprocess_environment_keeps_only_the_trusted_nvidia_parent_key() -> None:
    provider = _provider("nv_build", api_key="nvapi-x", base_url="https://integrate.api.nvidia.com/v1")
    configured = {
        "OPENAI_API_KEY": "openai-key",
        "OPENAI_BASE_URL": "https://api.openai.com/v1",
        "ANTHROPIC_API_KEY": "anthropic-key",
    }
    provider_env = {"NVIDIA_API_KEY": "nvapi-x"}

    opencode = _harbor_subprocess_environment(
        env_mode="local",
        provider=provider,
        configured_runtime_env=configured,
        provider_env=provider_env,
        agent="opencode",
        agent_model="nvidia/meta/llama-3.1-8b-instruct",
    )
    codex = _harbor_subprocess_environment(
        env_mode="local",
        provider=provider,
        configured_runtime_env=configured,
        provider_env=provider_env,
        agent="codex",
        agent_model="nvidia/nemotron-3-nano-30b-a3b",
    )
    claude = _harbor_subprocess_environment(
        env_mode="local",
        provider=provider,
        configured_runtime_env=configured,
        provider_env=provider_env,
        agent="claude-code",
        agent_model="nvidia/nemotron-3-nano-30b-a3b",
    )

    assert opencode["OPENAI_API_KEY"] == "nvapi-x"
    assert opencode["OPENAI_BASE_URL"] == "https://integrate.api.nvidia.com/v1"
    assert "ANTHROPIC_API_KEY" not in opencode
    assert codex["NVIDIA_API_KEY"] == "nvapi-x"
    assert "OPENAI_API_KEY" not in codex
    assert "OPENAI_BASE_URL" not in codex
    assert "ANTHROPIC_API_KEY" not in codex
    assert claude["NVIDIA_API_KEY"] == "nvapi-x"
    assert "ANTHROPIC_API_KEY" not in claude
    assert "OPENAI_API_KEY" not in claude
    assert "OPENAI_BASE_URL" not in claude


def test_local_host_env_excludes_provider_keys_by_default(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setenv("NVIDIA_API_KEY", "nvapi-secret")
    monkeypatch.setenv("OPENAI_API_KEY", "sk-secret")
    env = SkillEvaluatorLocalEnvironment._local_host_env(inherit_agent_keys=False)
    assert "NVIDIA_API_KEY" not in env
    assert "OPENAI_API_KEY" not in env
    assert "PATH" in env


def test_local_host_env_inherits_on_opt_in(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setenv("NVIDIA_API_KEY", "nvapi-secret")
    env = SkillEvaluatorLocalEnvironment._local_host_env(inherit_agent_keys=True)
    assert env["NVIDIA_API_KEY"] == "nvapi-secret"


def test_default_runtime_root_rejects_host_home(monkeypatch: pytest.MonkeyPatch) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    monkeypatch.setenv("SKILLEVALUATOR_RUNTIME_DIR", str(Path.home()))

    with pytest.raises(ValueError, match="dedicated subdirectory"):
        local_runtime.default_runtime_root()


def test_runtime_read_binds_reject_host_home(tmp_path: Path) -> None:
    environment = _local_environment(tmp_path)
    environment._runtime_root = Path.home()

    with pytest.raises(ValueError, match="dedicated subdirectory"):
        environment._runtime_ro_binds()


def test_background_server_is_rejected_even_with_declared_ports(tmp_path: Path) -> None:
    environment = _local_environment(tmp_path)
    command = "python -m http.server 8000 &"

    reason = environment._local_command_guardrail_reason(
        command,
        command,
        {"HARBOR_DECLARED_PORTS": "8000"},
    )

    assert "unsupported in local mode" in reason
    assert "Docker" in reason


@pytest.mark.parametrize(
    "command",
    [
        "setsid sh -c 'curl https://example.com &'",
        "bash -c 'setsid sleep 60'",
        "bash -lc 'nohup sleep 60'",
        "env -i setsid sleep 60",
        "env -i SAFE=1 nohup sleep 60",
        "nice -n 5 setsid sleep 60",
        "nice nohup sleep 60",
    ],
)
def test_detached_setsid_process_is_rejected_before_launch(tmp_path: Path, command: str) -> None:
    environment = _local_environment(tmp_path)

    reason = environment._local_command_guardrail_reason(
        command,
        command,
        {"HARBOR_DECLARED_PORTS": "443"},
    )

    assert "detached" in reason
    assert "Docker" in reason


def test_detached_launcher_word_as_plain_argument_is_not_rejected(tmp_path: Path) -> None:
    environment = _local_environment(tmp_path)
    command = "printf '%s' daemon"

    assert environment._local_command_guardrail_reason(command, command, {}) == ""


def test_quoted_url_ampersand_is_not_treated_as_background_operator(tmp_path: Path) -> None:
    environment = _local_environment(tmp_path)
    command = 'printf %s "https://example.com/query?a=1&b=2"'

    reason = environment._local_command_guardrail_reason(command, command, {})

    assert reason == ""


@pytest.mark.parametrize(
    "command",
    (
        "sleep 30 & printf wait",
        "sleep 30 & wait",
        "bash -c '(sleep 30) >/dev/null 2>&1 &'",
        "sh -lc 'sleep 30 & wait'",
    ),
)
def test_background_command_cannot_bypass_guard_with_wait_argument(tmp_path: Path, command: str) -> None:
    environment = _local_environment(tmp_path)

    reason = environment._local_command_guardrail_reason(command, command, {})

    assert "unsupported in local mode" in reason


def test_nested_background_shell_is_blocked_before_process_survives(tmp_path: Path) -> None:
    environment = _local_environment(tmp_path)
    marker = environment._workspace / "nested-background-survived"
    command = "bash -c '(sleep .2; printf survived > nested-background-survived) >/dev/null 2>&1 &'"

    async def run_probe() -> object:
        result = await environment.exec(command)
        await asyncio.sleep(0.3)
        return result

    result = asyncio.run(run_probe())

    assert result.return_code == 126
    assert not marker.exists()


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@pytest.mark.parametrize(
    "name",
    [
        "SAFE_TASK_VALUE",
        "HARBOR_INSTRUCTION",
        "HARBOR_CLAUDE_CODE_INSTRUCTION_0123456789ABCDEF0123456789ABCDEF",
        "CLAUDE_CODE_INSTRUCTION_0123456789ABCDEF0123456789ABCDEF",
    ],
)
def test_task_env_is_hidden_from_launcher_but_reaches_inner_command(tmp_path: Path, name: str) -> None:
    environment = _local_environment(tmp_path)
    captured_env = tmp_path / "launcher-env.json"
    launcher = tmp_path / "capture-launcher.py"
    launcher.write_text(
        "import json, os, sys\n"
        "from pathlib import Path\n"
        "capture = {\n"
        "    'environment': dict(os.environ),\n"
        "    'payload_files': sorted(path.name for path in Path(sys.argv[2]).glob('.command-env-*')),\n"
        "}\n"
        "Path(sys.argv[1]).write_text(json.dumps(capture), encoding='utf-8')\n"
        "os.execvp(sys.argv[3], sys.argv[3:])\n",
        encoding="utf-8",
    )

    class CaptureLauncher:
        plan = local_sandbox.SandboxPlan("none", "advisory-only", "capture")

        @staticmethod
        def wrap(argv: list[str], **_kwargs: object) -> list[str]:
            return [sys.executable, str(launcher), str(captured_env), str(environment._tmp), *argv]

    environment._sandbox = CaptureLauncher()

    instruction = "inner-only\nKeep 'quotes', \"double quotes\", $(printf literal), `printf literal`, and π."
    result = asyncio.run(environment.exec(f'printf %s "${name}"', env={name: instruction}))

    assert result.return_code == 0, result.stderr
    assert result.stdout == instruction
    capture = json.loads(captured_env.read_text(encoding="utf-8"))
    launcher_env = capture["environment"]
    assert name not in launcher_env
    assert capture["payload_files"] == []


@pytest.mark.parametrize(
    "name",
    [
        "HARBOR_INSTRUCTION",
        "HARBOR_CLAUDE_CODE_INSTRUCTION_0123456789ABCDEF0123456789ABCDEF",
    ],
)
def test_local_instruction_env_preserves_protected_paths(tmp_path: Path, name: str) -> None:
    environment = _local_environment(tmp_path)
    supplied = {
        name: "Read /workspace/SKILL.md",
        "HARBOR_WORKSPACE_DIR": "/outside",
        "HARBOR_GRADER": "/outside/grader.py",
        "HOME": "/outside",
        "TMPDIR": "/outside",
        "PATH": "/outside",
        "HARBOR_DECLARED_PORTS": "8000",
    }

    child_env = environment._exec_env(supplied)

    assert child_env[name] == f"Read {environment._workspace / 'SKILL.md'}"
    assert child_env["HARBOR_WORKSPACE_DIR"] == str(environment._workspace)
    assert child_env["HARBOR_GRADER"] == str(environment._tests / "grader.py")
    assert child_env["HOME"] == str(environment._home)
    assert child_env["TMPDIR"] == str(environment._tmp)
    assert "/outside" not in child_env["PATH"]
    assert child_env["HARBOR_DECLARED_PORTS"] == "8000"


@pytest.mark.parametrize(
    "name",
    [
        "HARBOR_UNKNOWN",
        "HARBOR_INSTRUCTION_EXTRA",
        "HARBOR_CLAUDE_CODE_INSTRUCTION_",
        "HARBOR_CLAUDE_CODE_INSTRUCTION_ABCDEF",
        "HARBOR_CLAUDE_CODE_INSTRUCTION_" + "A" * 31,
        "HARBOR_CLAUDE_CODE_INSTRUCTION_" + "A" * 33,
        "HARBOR_CLAUDE_CODE_INSTRUCTION_" + "G" * 32,
        "HARBOR_CLAUDE_CODE_INSTRUCTION_" + "a" * 32,
        "HARBOR_OTHER_INSTRUCTION_" + "A" * 32,
        "HARBOR_CLAUDE_CODE_INSTRUCTION_" + "A" * 32 + "_PATH",
        "HARBOR_CLAUDE_CODE_INSTRUCTION_" + "A" * 32 + "\n",
    ],
)
def test_local_instruction_env_does_not_allow_unrecognized_harbor_values(tmp_path: Path, name: str) -> None:
    environment = _local_environment(tmp_path)

    assert name not in environment._exec_env({name: "untrusted"})


def test_local_claude_instruction_env_preserves_literal_prompt_paths(tmp_path: Path) -> None:
    environment = _local_environment(tmp_path)
    name = "CLAUDE_CODE_INSTRUCTION_0123456789ABCDEF0123456789ABCDEF"
    instruction = 'Read /workspace/SKILL.md and /tests/test.sh; keep "quotes", $(printf literal), and π.\n'

    child_env = environment._exec_env({name: instruction, "HOME": "/outside"})

    assert child_env[name] == instruction
    assert child_env["HOME"] == str(environment._home)


@pytest.mark.parametrize(
    "name",
    [
        "CLAUDE_CODE_INSTRUCTION",
        "CLAUDE_CODE_INSTRUCTION_",
        "CLAUDE_CODE_INSTRUCTION_ABCDEF",
        "CLAUDE_CODE_INSTRUCTION_" + "A" * 31,
        "CLAUDE_CODE_INSTRUCTION_" + "A" * 33,
        "CLAUDE_CODE_INSTRUCTION_" + "G" * 32,
        "CLAUDE_CODE_INSTRUCTION_" + "a" * 32,
        "CLAUDE_CODE_INSTRUCTION_" + "A" * 32 + "_PATH",
        "CLAUDE_CODE_INSTRUCTION_" + "A" * 32 + "\n",
    ],
)
def test_local_instruction_env_rejects_malformed_claude_channels(tmp_path: Path, name: str) -> None:
    environment = _local_environment(tmp_path)

    assert name not in environment._exec_env({name: "untrusted"})


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@pytest.mark.parametrize("sandbox_mode", ["off", pytest.param("require", marks=pytest.mark.integration)])
def test_harbor_claude_run_delivers_instruction_to_local_cli(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path, sandbox_mode: str
) -> None:
    """Run the pinned Harbor producer and local subprocess boundary without a model call."""
    environment = _initialized_local_environment(tmp_path)
    environment._runtime_agent = "claude-code"
    environment._sandbox_mode = sandbox_mode
    if sandbox_mode == "require":
        try:
            local_sandbox.detect("require")
        except local_sandbox.SandboxUnavailable as exc:
            pytest.skip(str(exc))
    monkeypatch.setenv("ANTHROPIC_API_KEY", "test-only")
    monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://provider.example/v1")
    instruction = (
        "Explain argparse's nargs='?' in two lines.\n"
        "Keep /workspace/SKILL.md, /tests/test.sh, --permission-mode=bypassPermissions, "
        '"quotes", $(touch injected), `touch injected`, and π.\n'
    )
    agent = SkillEvaluatorLocalClaudeCode(logs_dir=tmp_path / "agent", model_name="anthropic/test-model")

    async def exercise() -> None:
        await environment.start()
        try:
            cli = environment._home / ".local" / "bin" / "claude"
            cli.parent.mkdir(parents=True, exist_ok=True)
            cli.write_text(
                f"#!{sys.executable}\n"
                "import json, os, sys\n"
                "json.dump({'prompt': sys.stdin.read(), 'argv': sys.argv[1:], "
                "'instruction_env': [key for key in os.environ if key.startswith("
                "('CLAUDE_CODE_INSTRUCTION_', 'claude_code_instruction_', 'HARBOR_CLAUDE_CODE_INSTRUCTION_'))]}, sys.stdout)\n",
                encoding="utf-8",
            )
            cli.chmod(0o755)
            await agent.run(instruction, environment=environment, context=object())
        finally:
            await environment.stop(delete=False)

    asyncio.run(exercise())

    observed = json.loads((environment.trial_paths.agent_dir / "claude-code.txt").read_text())
    assert observed["prompt"] == instruction
    assert "--print" in observed["argv"]
    assert "--permission-mode=auto" in observed["argv"]
    assert observed["instruction_env"] == []
    assert not (environment._workspace / "injected").exists()


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_real_local_exec_streams_stdout_and_stderr_through_harbor_callback(tmp_path: Path) -> None:
    environment = _initialized_local_environment(tmp_path)
    callback_chunks: list[tuple[str, str]] = []

    async def on_output(text: str, stream: str) -> None:
        callback_chunks.append((text, stream))

    async def exercise() -> object:
        await environment.start()
        with environment.scoped_output_callback(on_output):
            return await environment.exec("printf stdout-value; printf stderr-value >&2")

    result = asyncio.run(exercise())

    assert result.stdout == "stdout-value"
    assert result.stderr == "stderr-value"
    assert "".join(text for text, stream in callback_chunks if stream == "stdout") == result.stdout
    assert "".join(text for text, stream in callback_chunks if stream == "stderr") == result.stderr


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_local_streamed_nonzero_exit_preserves_output_and_return_code(tmp_path: Path) -> None:
    environment = _initialized_local_environment(tmp_path)
    callback_chunks: list[tuple[str, str]] = []
    secret = "nonzero-stream-secret"

    async def on_output(text: str, stream: str) -> None:
        callback_chunks.append((text, stream))

    async def exercise() -> object:
        await environment.start()
        with environment.scoped_output_callback(on_output):
            return await environment.exec(
                'printf "stdout=%s\\n" "$NONZERO_TOKEN"; printf "stderr=%s\\n" "$NONZERO_TOKEN" >&2; exit 7',
                env={"NONZERO_TOKEN": secret},
            )

    result = asyncio.run(exercise())
    callback_stdout = "".join(text for text, stream in callback_chunks if stream == "stdout")
    callback_stderr = "".join(text for text, stream in callback_chunks if stream == "stderr")

    assert result.return_code == 7
    assert callback_stdout == result.stdout
    assert callback_stderr == result.stderr
    assert secret not in callback_stdout
    assert secret not in callback_stderr


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@pytest.mark.parametrize(
    "proxy_uri",
    [
        "https://local-user:[email protected]:8443",
        "https://local%2Duser:local%[email protected]:8443",
        "local-user:[email protected]:8443",
    ],
    ids=["uri", "percent-encoded", "schemeless"],
)
def test_local_nonzero_exit_redacts_detached_proxy_components_from_callback_and_result(
    tmp_path: Path,
    proxy_uri: str,
) -> None:
    environment = _initialized_local_environment(tmp_path)
    callback_chunks: list[tuple[str, str]] = []

    async def on_output(text: str, stream: str) -> None:
        callback_chunks.append((text, stream))

    async def exercise() -> object:
        await environment.start()
        with environment.scoped_output_callback(on_output):
            return await environment.exec(
                "printf 'proxy rejected local-user with local-password\\n'; exit 7",
                env={"HTTPS_PROXY": proxy_uri},
            )

    result = asyncio.run(exercise())
    rendered = "".join(text for text, _stream in callback_chunks) + (result.stdout or "") + (result.stderr or "")

    assert result.return_code == 7
    assert "local-user" not in rendered
    assert "local-password" not in rendered


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_real_local_exec_redacts_merged_secrets_across_byte_and_line_boundaries(
    tmp_path: Path,
) -> None:
    persistent_secret = "persistent-first\npersistent-second"
    scoped_secret = "scoped-secret-value"
    per_call_secret = "per-call-secret-value"
    environment = _initialized_local_environment(
        tmp_path,
        persistent_env={"PERSISTENT_TOKEN": persistent_secret},
    )
    callback_chunks: list[tuple[str, str]] = []
    script = """
import os
import time

values = (
    (1, os.environ["PERSISTENT_TOKEN"]),
    (2, os.environ["SCOPED_SECRET"]),
    (1, os.environ["PER_CALL_KEY"]),
    (2, "unicode-snowman-☃"),
)
for descriptor, value in values:
    payload = value.encode("utf-8")
    for byte in payload:
        os.write(descriptor, bytes((byte,)))
        time.sleep(0.001)
    os.write(descriptor, b"\\n")
"""
    command = f"{shlex.quote(sys.executable)} -c {shlex.quote(script)}"

    async def on_output(text: str, stream: str) -> None:
        callback_chunks.append((text, stream))

    async def exercise() -> object:
        await environment.start()
        with (
            environment.scoped_exec_env({"SCOPED_SECRET": scoped_secret}),
            environment.scoped_output_callback(on_output),
        ):
            return await environment.exec(command, env={"PER_CALL_KEY": per_call_secret})

    result = asyncio.run(exercise())
    callback_stdout = "".join(text for text, stream in callback_chunks if stream == "stdout")
    callback_stderr = "".join(text for text, stream in callback_chunks if stream == "stderr")

    assert callback_stdout == result.stdout
    assert callback_stderr == result.stderr
    assert "unicode-snowman-☃" in callback_stderr
    for secret in (persistent_secret, scoped_secret, per_call_secret):
        assert secret not in callback_stdout
        assert secret not in callback_stderr
        assert secret not in (result.stdout or "")
        assert secret not in (result.stderr or "")


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_local_stream_redaction_handles_short_sensitive_and_marker_collision_values(
    tmp_path: Path,
) -> None:
    environment = _initialized_local_environment(tmp_path)
    callback_chunks: list[str] = []

    async def on_output(text: str, _stream: str) -> None:
        callback_chunks.append(text)

    async def exercise() -> object:
        await environment.start()
        with environment.scoped_output_callback(on_output):
            return await environment.exec(
                'printf "%s|%s|%s" "$API_KEY" "$PUBLIC_LABEL" "$COLLISION_SECRET"',
                env={
                    "API_KEY": "x",
                    "PUBLIC_LABEL": "ok",
                    "COLLISION_SECRET": "redacted",
                },
            )

    result = asyncio.run(exercise())
    callback_output = "".join(callback_chunks)

    assert callback_output == result.stdout
    assert "|ok|" in callback_output
    assert "x" not in callback_output
    assert "redacted" not in callback_output.lower()


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_local_stream_redacts_short_compact_credential_names_without_substring_false_positives(
    tmp_path: Path,
) -> None:
    environment = _initialized_local_environment(tmp_path)
    callback_chunks: list[str] = []

    async def on_output(text: str, _stream: str) -> None:
        callback_chunks.append(text)

    async def exercise() -> object:
        await environment.start()
        with environment.scoped_output_callback(on_output):
            return await environment.exec(
                'printf "%s|%s|%s|%s" "$MYSECRET" "$AUTHENTICATION" "$MONKEY" "$KEYBOARD"',
                env={
                    "MYSECRET": "pw",
                    "AUTHENTICATION": "id",
                    "MONKEY": "banana",
                    "KEYBOARD": "clacky",
                },
            )

    result = asyncio.run(exercise())
    callback_output = "".join(callback_chunks)

    assert callback_output == result.stdout
    assert "pw" not in callback_output
    assert "id" not in callback_output
    assert callback_output.endswith("banana|clacky")


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_local_stream_redacts_known_secret_patterns_across_reader_chunks(tmp_path: Path) -> None:
    environment = _initialized_local_environment(tmp_path)
    callback_chunks: list[tuple[str, str]] = []
    sk_secret = "".join(("sk-", "Ab1Cd2Ef3Gh4Ij5Kl6Mn7Op8"))  # noqa: FLY002
    nvapi_secret = "".join(("nvapi-", "Ab1Cd2Ef3Gh4Ij5Kl6Mn7Op8"))  # noqa: FLY002
    crsr_secret = "".join(("crsr_", "0123456789abcdef"))  # noqa: FLY002
    openshift_secret = "".join(("sha256~", "Abc.def_Ghi-jkl~mno"))  # noqa: FLY002
    jwt_secret = ".".join(("eyJ" + "A" * 20, "B" * 20, "C" * 20))
    script = f"""
import os
import time

values = (
    (1, {sk_secret!r}),
    (2, {nvapi_secret!r}),
    (1, {crsr_secret!r}),
    (2, {openshift_secret!r}),
    (1, {jwt_secret!r}),
    (2, "task-granularity"),
)
for descriptor, value in values:
    for byte in value.encode("utf-8"):
        os.write(descriptor, bytes((byte,)))
        time.sleep(0.001)
    os.write(descriptor, b"\\n")
"""
    command = f"{shlex.quote(sys.executable)} -c {shlex.quote(script)}"

    async def on_output(text: str, stream: str) -> None:
        callback_chunks.append((text, stream))

    async def exercise() -> object:
        await environment.start()
        with environment.scoped_output_callback(on_output):
            return await environment.exec(command)

    result = asyncio.run(exercise())
    callback_stdout = "".join(text for text, stream in callback_chunks if stream == "stdout")
    callback_stderr = "".join(text for text, stream in callback_chunks if stream == "stderr")

    assert callback_stdout == result.stdout
    assert callback_stderr == result.stderr
    assert "task-granularity" in callback_stderr
    for secret in (sk_secret, nvapi_secret, crsr_secret, openshift_secret, jwt_secret):
        assert secret not in callback_stdout
        assert secret not in callback_stderr


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_local_concurrent_callback_contexts_are_isolated(tmp_path: Path) -> None:
    environment = _initialized_local_environment(tmp_path)
    labels = tuple(f"callback-{index}" for index in range(10))
    callback_outputs: dict[str, list[str]] = {label: [] for label in labels}

    async def exercise() -> dict[str, object]:
        await environment.start()

        async def run(label: str) -> object:
            async def on_output(text: str, _stream: str) -> None:
                callback_outputs[label].append(text)

            with environment.scoped_output_callback(on_output):
                return await environment.exec(f"printf '{label}-first\\n'; sleep 0.05; printf '{label}-second\\n'")

        results = await asyncio.gather(*(run(label) for label in labels))
        return dict(zip(labels, results, strict=True))

    results = asyncio.run(exercise())

    for label in labels:
        rendered = "".join(callback_outputs[label])
        assert rendered == results[label].stdout
        for other_label in labels:
            if other_label != label:
                assert other_label not in rendered


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_local_nested_callbacks_run_in_scope_order(tmp_path: Path) -> None:
    environment = _initialized_local_environment(tmp_path)
    calls: list[tuple[str, str, str]] = []

    async def outer(text: str, stream: str) -> None:
        calls.append(("outer", text, stream))

    async def inner(text: str, stream: str) -> None:
        calls.append(("inner", text, stream))

    async def exercise() -> object:
        await environment.start()
        with (
            environment.scoped_output_callback(outer),
            environment.scoped_output_callback(inner),
        ):
            return await environment.exec("printf 'nested-output\\n'")

    result = asyncio.run(exercise())

    assert result.stdout == "nested-output\n"
    assert calls == [
        ("outer", "nested-output\n", "stdout"),
        ("inner", "nested-output\n", "stdout"),
    ]


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_local_callback_receives_complete_line_before_process_exit(tmp_path: Path) -> None:
    environment = _initialized_local_environment(tmp_path)
    first_line = asyncio.Event()

    async def on_output(text: str, stream: str) -> None:
        if text == "first-line\n" and stream == "stdout":
            first_line.set()

    async def exercise() -> object:
        await environment.start()
        with environment.scoped_output_callback(on_output):
            task = asyncio.create_task(environment.exec("printf 'first-line\\n'; sleep 1; printf done"))
            await asyncio.wait_for(first_line.wait(), timeout=0.5)
            assert not task.done()
            return await task

    result = asyncio.run(exercise())

    assert result.stdout == "first-line\ndone"


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_local_callback_streams_safe_partial_line_before_process_exit(tmp_path: Path) -> None:
    environment = _initialized_local_environment(tmp_path)
    partial_output = asyncio.Event()
    callback_chunks: list[str] = []

    async def on_output(text: str, stream: str) -> None:
        if stream == "stdout":
            callback_chunks.append(text)
            if "safe-partial-output" in "".join(callback_chunks):
                partial_output.set()

    async def exercise() -> object:
        await environment.start()
        with environment.scoped_output_callback(on_output):
            task = asyncio.create_task(environment.exec("printf safe-partial-output; sleep 1; printf done"))
            await asyncio.wait_for(partial_output.wait(), timeout=0.5)
            assert not task.done()
            return await task

    result = asyncio.run(exercise())

    assert "".join(callback_chunks) == result.stdout == "safe-partial-outputdone"


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_local_streaming_preserves_exact_json_stdin_bootstrap(tmp_path: Path) -> None:
    environment = _initialized_local_environment(tmp_path)
    captured_payloads: list[bytes] = []
    original_collect = environment._collect_streamed_output
    per_call_env = {"PER_CALL_VALUE": "☃"}

    async def capture_payload(
        proc: asyncio.subprocess.Process,
        stdin_data: bytes,
        callback_output: object,
    ) -> tuple[bytes, bytes]:
        captured_payloads.append(stdin_data)
        return await original_collect(
            proc,
            stdin_data,
            callback_output,  # type: ignore[arg-type]
        )

    async def on_output(_text: str, _stream: str) -> None:
        return None

    async def exercise() -> tuple[object, bytes]:
        await environment.start()
        with (
            environment.scoped_exec_env({"SCOPED_VALUE": "scoped"}),
            environment.scoped_output_callback(on_output),
        ):
            expected_payload = json.dumps(environment._exec_env(per_call_env)).encode("utf-8")
            environment._collect_streamed_output = capture_payload  # type: ignore[method-assign]
            result = await environment.exec('printf %s "$PER_CALL_VALUE"', env=per_call_env)
        return result, expected_payload

    result, expected_payload = asyncio.run(exercise())

    assert result.stdout == "☃"
    assert captured_payloads == [expected_payload]


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_local_exec_without_callback_uses_bounded_stream_collector(tmp_path: Path) -> None:
    environment = _initialized_local_environment(tmp_path)
    original_collect = environment._collect_streamed_output
    callback_outputs: list[object] = []

    async def capture_stream(
        proc: asyncio.subprocess.Process,
        stdin_data: bytes,
        callback_output: object,
    ) -> tuple[bytes, bytes]:
        callback_outputs.append(callback_output)
        return await original_collect(
            proc,
            stdin_data,
            callback_output,  # type: ignore[arg-type]
        )

    async def exercise() -> object:
        await environment.start()
        environment._collect_streamed_output = capture_stream  # type: ignore[method-assign]
        return await environment.exec("printf bounded-only")

    result = asyncio.run(exercise())

    assert callback_outputs == [None]
    assert result.stdout == "bounded-only"
    assert result.stderr == ""
    assert result.return_code == 0


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_local_streaming_tolerates_child_closing_json_stdin_early(tmp_path: Path) -> None:
    environment = _initialized_local_environment(tmp_path)

    class EarlyExitSandbox:
        plan = local_sandbox.SandboxPlan("none", "advisory-only", "early-exit-test")

        @staticmethod
        def wrap(_argv: list[str], **_kwargs: object) -> list[str]:
            return [sys.executable, "-c", "import os; os._exit(7)"]

    environment._sandbox = EarlyExitSandbox()

    async def on_output(_text: str, _stream: str) -> None:
        return None

    async def exercise() -> object:
        await environment.start()
        environment._sandbox = EarlyExitSandbox()
        with environment.scoped_output_callback(on_output):
            return await environment.exec("ignored", env={"FILLER": "v" * (1024 * 1024)})

    result = asyncio.run(exercise())

    assert result.return_code == 7
    assert result.stdout == ""
    assert result.stderr == ""


@pytest.mark.skipif(os.name != "posix", reason="requires POSIX process groups")
@pytest.mark.parametrize("error_type", [TimeoutError, _LocalCallbackBaseError, asyncio.CancelledError])
def test_local_callback_exception_is_propagated_after_process_reap(
    tmp_path: Path,
    error_type: type[BaseException],
) -> None:
    environment = _initialized_local_environment(tmp_path)
    callback_error = error_type("local callback failed")
    secret = "local-callback-base-error-secret"
    processes: list[asyncio.subprocess.Process] = []
    create_subprocess_exec = asyncio.create_subprocess_exec
    callback_chunks: list[str] = []
    reaped_by_exec: list[bool] = []
    caught_errors: list[BaseException] = []

    async def capture_process(*args: object, **kwargs: object) -> asyncio.subprocess.Process:
        process = await create_subprocess_exec(*args, **kwargs)
        processes.append(process)
        return process

    async def failing_callback(text: str, _stream: str) -> None:
        callback_chunks.append(text)
        raise callback_error

    async def exercise() -> None:
        await environment.start()
        try:
            with (
                pytest.MonkeyPatch.context() as patch,
                environment.scoped_output_callback(failing_callback),
            ):
                patch.setattr(asyncio, "create_subprocess_exec", capture_process)
                for _ in range(3):
                    with pytest.raises(error_type) as caught:
                        await environment.exec(
                            'printf "%s\\n" "$CALLBACK_SECRET"; sleep 30',
                            env={"CALLBACK_SECRET": secret},
                        )
                    caught_errors.append(caught.value)
                    reaped_by_exec.append(processes[-1].returncode is not None)
        finally:
            for process in processes:
                if process.returncode is None:
                    with contextlib.suppress(ProcessLookupError):
                        os.killpg(process.pid, signal.SIGKILL)
                    await process.wait()

    asyncio.run(exercise())

    assert callback_chunks
    assert secret not in "".join(callback_chunks)
    assert secret not in str(callback_error)
    assert caught_errors == [callback_error] * 3
    assert len(processes) == 3
    assert reaped_by_exec == [True] * 3


@pytest.mark.skipif(os.name != "posix", reason="requires POSIX process groups")
def test_local_callback_error_remains_primary_when_cleanup_reports_failure(tmp_path: Path) -> None:
    environment = _initialized_local_environment(tmp_path)
    callback_error = _LocalCallbackBaseError("primary callback failure")
    secret = "synthetic cleanup report failure"
    original_terminate = environment._terminate_process_tree
    cleanup_calls = 0

    async def failing_cleanup(
        _proc: asyncio.subprocess.Process,
        _communication: asyncio.Task[tuple[bytes, bytes]] | None = None,
    ) -> tuple[bytes, bytes]:
        nonlocal cleanup_calls
        cleanup_calls += 1
        raise PermissionError(errno.EACCES, "Denied", secret)

    async def failing_callback(_text: str, _stream: str) -> None:
        raise callback_error

    async def exercise() -> tuple[BaseException, bool, bool]:
        await environment.start()
        environment._terminate_process_tree = failing_cleanup  # type: ignore[method-assign]
        with (
            environment.scoped_output_callback(failing_callback),
            pytest.raises(_LocalCallbackBaseError) as caught,
        ):
            await environment.exec(
                "printf 'callback-output\\n'; sleep 30",
                env={"API_KEY": secret},
            )
        retained_before_retry = bool(environment._active_processes)
        environment._terminate_process_tree = original_terminate  # type: ignore[method-assign]
        await environment.stop(delete=False)
        return caught.value, retained_before_retry, not environment._active_processes

    caught, retained_before_retry, released_after_retry = asyncio.run(exercise())

    assert caught is callback_error
    assert isinstance(caught.__cause__, RuntimeError)
    assert caught.__cause__.__context__ is None
    assert cleanup_calls == 1
    assert retained_before_retry
    assert released_after_retry
    assert any("cleanup" in note.lower() for note in caught.__notes__)
    assert secret not in "".join(caught.__notes__)
    assert secret not in str(caught.__cause__)
    assert secret not in "".join(traceback.format_exception(caught))


@pytest.mark.skipif(os.name != "posix", reason="requires POSIX process groups")
def test_local_callback_error_message_receives_only_redacted_output(tmp_path: Path) -> None:
    environment = _initialized_local_environment(tmp_path)
    secret = "callback-error-message-secret"

    async def failing_callback(text: str, _stream: str) -> None:
        raise RuntimeError(f"consumer rejected: {text}")

    async def exercise() -> RuntimeError:
        await environment.start()
        with (
            environment.scoped_output_callback(failing_callback),
            pytest.raises(RuntimeError, match="consumer rejected") as caught,
        ):
            await environment.exec(
                'printf "%s\\n" "$ERROR_TOKEN"; sleep 30',
                env={"ERROR_TOKEN": secret},
            )
        return caught.value

    error = asyncio.run(exercise())

    assert secret not in str(error)
    assert "consumer rejected" in str(error)


@pytest.mark.skipif(os.name != "posix", reason="requires POSIX process groups")
def test_local_stream_collector_failure_is_propagated_after_process_reap(tmp_path: Path) -> None:
    environment = _initialized_local_environment(tmp_path)
    collector_error = RuntimeError("synthetic stream collector failure")
    processes: list[asyncio.subprocess.Process] = []
    create_subprocess_exec = asyncio.create_subprocess_exec
    reaped_by_exec: list[bool] = []

    async def capture_process(*args: object, **kwargs: object) -> asyncio.subprocess.Process:
        process = await create_subprocess_exec(*args, **kwargs)
        processes.append(process)
        return process

    async def failing_collector(*_args: object, **_kwargs: object) -> tuple[bytes, bytes]:
        raise collector_error

    async def on_output(_text: str, _stream: str) -> None:
        return None

    async def exercise() -> BaseException:
        await environment.start()
        environment._collect_streamed_output = failing_collector  # type: ignore[method-assign]
        try:
            with (
                pytest.MonkeyPatch.context() as patch,
                environment.scoped_output_callback(on_output),
                pytest.raises(RuntimeError, match="synthetic stream collector failure") as caught,
            ):
                patch.setattr(asyncio, "create_subprocess_exec", capture_process)
                await environment.exec("sleep 30")
            reaped_by_exec.append(processes[0].returncode is not None)
            return caught.value
        finally:
            for process in processes:
                if process.returncode is None:
                    with contextlib.suppress(ProcessLookupError):
                        os.killpg(process.pid, signal.SIGKILL)
                    await process.wait()

    caught = asyncio.run(exercise())

    assert caught is collector_error
    assert caught.__cause__ is None
    assert reaped_by_exec == [True]


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_exec_forwards_strict_read_policy_to_sandbox(tmp_path: Path) -> None:
    environment = _local_environment(tmp_path)
    environment._strict_reads = True
    captured: dict[str, object] = {}

    class CaptureSandbox:
        plan = local_sandbox.SandboxPlan("none", "advisory-only", "capture")

        @staticmethod
        def wrap(argv: list[str], **kwargs: object) -> list[str]:
            captured.update(kwargs)
            return argv

    environment._sandbox = CaptureSandbox()

    result = asyncio.run(environment.exec("printf ok"))

    assert result.return_code == 0
    assert captured["strict_reads"] is True


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@pytest.mark.parametrize("strict_reads", [False, True])
def test_seatbelt_exec_uses_canonical_interpreter_for_sandbox_bootstrap(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
    strict_reads: bool,
) -> None:
    venv_python = tmp_path / "venv" / "bin" / "python"
    venv_python.parent.mkdir(parents=True)
    venv_python.symlink_to(Path(sys.executable).resolve())
    monkeypatch.setattr(sys, "executable", str(venv_python))
    environment = _local_environment(tmp_path)
    environment._strict_reads = strict_reads
    captured: dict[str, object] = {}

    class CaptureSandbox:
        plan = local_sandbox.SandboxPlan("seatbelt", "kernel-macos", "capture")

        @staticmethod
        def wrap(argv: list[str], **_kwargs: object) -> list[str]:
            captured["argv"] = argv
            return argv

    environment._sandbox = CaptureSandbox()

    result = asyncio.run(environment.exec("printf ok"))

    assert result.return_code == 0, result.stderr
    wrapped_argv = captured["argv"]
    assert isinstance(wrapped_argv, list)
    assert wrapped_argv[0] == str(Path(sys.executable).resolve())


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@pytest.mark.parametrize(
    ("backend", "strength", "strict_reads"),
    [
        ("bubblewrap", "kernel", True),
        ("none", "advisory-only", True),
    ],
)
def test_other_sandbox_modes_preserve_venv_interpreter_for_bootstrap(
    tmp_path: Path,
    backend: str,
    strength: str,
    strict_reads: bool,
) -> None:
    environment = _local_environment(tmp_path)
    environment._strict_reads = strict_reads
    captured: dict[str, object] = {}

    class CaptureSandbox:
        plan = local_sandbox.SandboxPlan(backend, strength, "capture")

        @staticmethod
        def wrap(argv: list[str], **_kwargs: object) -> list[str]:
            captured["argv"] = argv
            return argv

    environment._sandbox = CaptureSandbox()

    result = asyncio.run(environment.exec("printf ok"))

    assert result.return_code == 0, result.stderr
    wrapped_argv = captured["argv"]
    assert isinstance(wrapped_argv, list)
    assert wrapped_argv[0] == sys.executable


@pytest.mark.skipif(
    sys.platform != "darwin" or not Path("/usr/bin/sandbox-exec").exists(),
    reason="requires macOS Seatbelt",
)
def test_strict_exec_bootstrap_runs_from_fresh_private_tmp_venv_under_real_seatbelt(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    base_interpreter = next(
        (
            candidate
            for prefix in (Path("/opt/homebrew/bin"), Path("/usr/local/bin"))
            for version in ("3.13", "3.12")
            if (candidate := prefix / f"python{version}").exists()
        ),
        None,
    )
    if base_interpreter is None:
        pytest.skip("requires a Homebrew Python that creates a symlinked venv interpreter")

    with tempfile.TemporaryDirectory(prefix="skillevaluator-seatbelt-venv-", dir="/private/tmp") as temp_dir:
        venv_root = Path(temp_dir) / "venv"
        subprocess.run([str(base_interpreter), "-m", "venv", str(venv_root)], check=True, timeout=60)
        venv_python = venv_root / "bin" / "python"

        environment = _local_environment(tmp_path)
        environment._sandbox_mode = "require"
        environment._strict_reads = True
        environment._sandbox = local_sandbox.detect("require")

        with monkeypatch.context() as patch:
            patch.setattr(sys, "executable", str(venv_python))
            patch.setattr(sys, "prefix", str(venv_root))
            patch.setattr(sys, "exec_prefix", str(venv_root))
            result = asyncio.run(environment.exec("printf strict-bootstrap-ok"))

    assert result.return_code == 0, result.stderr
    assert result.stdout == "strict-bootstrap-ok"


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_exec_timeout_terminates_background_descendants(tmp_path: Path) -> None:
    # Deterministic under CPU load: the child sleeps far longer than the exec
    # timeout (so it can never legitimately write its marker), and instead of
    # trusting fixed sleeps the test polls until the recorded child PID is
    # gone. The previous 0.2s-timeout/0.5s-sleep pairing flaked under
    # pytest-xdist when scheduling latency ate the margins.
    environment = _local_environment(tmp_path)
    started = environment._workspace / "timeout-child-started"
    marker = environment._workspace / "timeout-child-survived"
    child_pid_path = environment._workspace / "timeout-child-pid"
    command = (
        "printf stdout-before-timeout; printf stderr-before-timeout >&2; "
        "printf started > timeout-child-started; "
        "(sleep 15 && printf survived > timeout-child-survived) & "
        "printf '%s' \"$!\" > timeout-child-pid; wait"
    )
    environment._local_command_guardrail_reason = lambda *_args: ""  # type: ignore[method-assign]

    async def run_timeout() -> object:
        return await environment.exec(command, timeout_sec=1.0)

    result = asyncio.run(run_timeout())

    assert result.return_code == 124
    assert result.stdout == "stdout-before-timeout"
    assert "stderr-before-timeout" in (result.stderr or "")
    assert "Timed out" in (result.stderr or "")
    assert started.exists(), "the background descendant did not start before the timeout"

    def process_exists(pid: int) -> bool:
        try:
            os.kill(pid, 0)
        except ProcessLookupError:
            return False
        except PermissionError:
            return True
        return True

    child_pid = int(child_pid_path.read_text())
    deadline = time.monotonic() + 10
    while process_exists(child_pid) and time.monotonic() < deadline:
        time.sleep(0.05)
    assert not process_exists(child_pid), "a background descendant survived the timeout kill"
    assert not marker.exists(), "a background descendant wrote after the command timed out"


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_streamed_timeout_callback_matches_result_and_contains_descendants(tmp_path: Path) -> None:
    environment = _initialized_local_environment(tmp_path)
    environment._local_command_guardrail_reason = lambda *_args: ""  # type: ignore[method-assign]
    secret = "stream-timeout-secret-value"
    child_pid_path = environment._workspace / "stream-timeout-child-pid"
    callback_chunks: list[tuple[str, str]] = []
    command = (
        'printf "%s\\n" "$TIMEOUT_TOKEN"; printf "stderr-before-timeout\\n" >&2; '
        "(sleep 30) & printf '%s' \"$!\" > stream-timeout-child-pid; wait"
    )

    async def on_output(text: str, stream: str) -> None:
        callback_chunks.append((text, stream))

    async def exercise() -> object:
        await environment.start()
        with environment.scoped_output_callback(on_output):
            return await environment.exec(
                command,
                env={"TIMEOUT_TOKEN": secret},
                timeout_sec=1,
            )

    result = asyncio.run(exercise())
    callback_stdout = "".join(text for text, stream in callback_chunks if stream == "stdout")
    callback_stderr = "".join(text for text, stream in callback_chunks if stream == "stderr")

    assert result.return_code == 124
    assert callback_stdout == result.stdout
    assert callback_stderr == result.stderr
    # macOS bash can report a background job's SIGTERM during group cleanup.
    assert re.fullmatch(
        r"stderr-before-timeout\n(?:bash: line \d+: +\d+ Terminated: 15 +\( sleep 30 \)\n)?Timed out",
        callback_stderr,
    )
    assert secret not in callback_stdout
    child_pid = int(child_pid_path.read_text(encoding="ascii"))
    _assert_process_exits(child_pid)


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_local_timeout_redacts_detached_proxy_components_from_callback_and_result(
    tmp_path: Path,
) -> None:
    environment = _initialized_local_environment(tmp_path)
    environment._local_command_guardrail_reason = lambda *_args: ""  # type: ignore[method-assign]
    callback_chunks: list[tuple[str, str]] = []

    async def on_output(text: str, stream: str) -> None:
        callback_chunks.append((text, stream))

    async def exercise() -> object:
        await environment.start()
        with environment.scoped_output_callback(on_output):
            return await environment.exec(
                "printf 'proxy rejected local-timeout-user with local-timeout-password\\n'; sleep 30",
                env={
                    "HTTPS_PROXY": "https://local-timeout-user:[email protected]:8443",
                },
                timeout_sec=0.1,
            )

    result = asyncio.run(exercise())
    rendered = "".join(text for text, _stream in callback_chunks) + (result.stdout or "") + (result.stderr or "")

    assert result.return_code == 124
    assert "local-timeout-user" not in rendered
    assert "local-timeout-password" not in rendered


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@pytest.mark.parametrize("with_callback", [False, True])
def test_timeout_diagnostic_cannot_synthesize_sensitive_value(
    tmp_path: Path,
    with_callback: bool,
) -> None:
    environment = _initialized_local_environment(tmp_path)
    secret = "Timed out"
    callback_chunks: list[tuple[str, str]] = []

    async def on_output(text: str, stream: str) -> None:
        callback_chunks.append((text, stream))

    async def exercise() -> object:
        await environment.start()
        callback_scope = environment.scoped_output_callback(on_output) if with_callback else contextlib.nullcontext()
        with callback_scope:
            return await environment.exec(
                "sleep 30",
                env={"TIMEOUT_SECRET": secret},
                timeout_sec=0.1,
            )

    result = asyncio.run(exercise())
    callback_stderr = "".join(text for text, stream in callback_chunks if stream == "stderr")

    assert result.return_code == 124
    assert secret not in (result.stderr or "")
    assert secret not in callback_stderr
    if with_callback:
        assert callback_stderr == result.stderr


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@pytest.mark.parametrize("with_callback", [False, True])
@pytest.mark.parametrize("secret", ["prefix\nTimed out", "\n"])
def test_timeout_diagnostic_uses_the_live_stderr_redactor_across_its_boundary(
    tmp_path: Path,
    with_callback: bool,
    secret: str,
) -> None:
    environment = _initialized_local_environment(tmp_path)
    callback_chunks: list[tuple[str, str]] = []

    async def on_output(text: str, stream: str) -> None:
        callback_chunks.append((text, stream))

    async def exercise() -> object:
        await environment.start()
        callback_scope = environment.scoped_output_callback(on_output) if with_callback else contextlib.nullcontext()
        with callback_scope:
            return await environment.exec(
                "printf prefix >&2; sleep 30",
                env={"TIMEOUT_SECRET": secret},
                timeout_sec=0.1,
            )

    result = asyncio.run(exercise())
    callback_stderr = "".join(text for text, stream in callback_chunks if stream == "stderr")

    assert result.return_code == 124
    assert secret not in (result.stderr or "")
    assert secret not in callback_stderr
    if with_callback:
        assert callback_stderr == result.stderr


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_blocked_callback_does_not_replace_command_timeout_with_cancelled_error(tmp_path: Path) -> None:
    environment = _initialized_local_environment(tmp_path)
    release_callback = asyncio.Event()

    async def blocked_callback(_text: str, _stream: str) -> None:
        await release_callback.wait()

    async def exercise() -> tuple[object, float]:
        await environment.start()
        started = time.monotonic()
        with environment.scoped_output_callback(blocked_callback):
            result = await environment.exec("printf callback-blocked; sleep 30", timeout_sec=0.1)
        return result, time.monotonic() - started

    result, elapsed = asyncio.run(exercise())

    assert elapsed < 3
    assert result.return_code == 124
    assert "Timed out" in (result.stderr or "")


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_callback_suppressing_one_cleanup_cancellation_is_not_reentered_or_leaked(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_environment

    environment = _initialized_local_environment(tmp_path)
    monkeypatch.setattr(local_environment, "_REAP_TERM_SECONDS", 0.01)
    monkeypatch.setattr(local_environment, "_REAP_KILL_SECONDS", 0.01)
    monkeypatch.setattr(local_environment, "_REAP_CANCEL_SECONDS", 0.05)
    callback_calls = 0

    async def cancellation_suppressing_callback(_text: str, _stream: str) -> None:
        nonlocal callback_calls
        callback_calls += 1
        try:
            await asyncio.Event().wait()
        except asyncio.CancelledError:
            # A callback can perform async cleanup after its first
            # cancellation. A bounded second cancellation must finish that
            # cleanup path; exec must never invoke it concurrently again.
            await asyncio.Event().wait()

    async def exercise() -> tuple[object, list[str]]:
        await environment.start()
        with environment.scoped_output_callback(cancellation_suppressing_callback):
            result = await asyncio.wait_for(
                environment.exec("printf callback-blocked; sleep 30", timeout_sec=0.02),
                timeout=0.5,
            )
        await asyncio.sleep(0)
        leaked = [
            repr(task.get_coro())
            for task in asyncio.all_tasks()
            if task is not asyncio.current_task()
            and not task.done()
            and any(
                name in repr(task.get_coro())
                for name in ("_collect_streamed_output", "invoke_callback", "_cancel_task_repeatedly")
            )
        ]
        return result, leaked

    result, leaked = asyncio.run(exercise())

    assert result.return_code == 124
    assert callback_calls == 1
    assert leaked == []


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@pytest.mark.parametrize("repeat_cancellation", [False, True])
def test_cancellation_during_timeout_diagnostic_callback_reaps_callback_tasks(
    tmp_path: Path,
    repeat_cancellation: bool,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_environment

    environment = _initialized_local_environment(tmp_path)
    monkeypatch.setattr(local_environment, "_REAP_TERM_SECONDS", 0.01)
    monkeypatch.setattr(local_environment, "_REAP_KILL_SECONDS", 0.01)
    monkeypatch.setattr(local_environment, "_REAP_CANCEL_SECONDS", 0.05)
    callback_started = asyncio.Event()
    callback_calls = 0

    async def blocked_callback(_text: str, _stream: str) -> None:
        nonlocal callback_calls
        callback_calls += 1
        callback_started.set()
        await asyncio.Event().wait()

    async def exercise() -> tuple[BaseException | None, list[str]]:
        await environment.start()
        with environment.scoped_output_callback(blocked_callback):
            task = asyncio.create_task(environment.exec("sleep 30", timeout_sec=0.02))
            await asyncio.wait_for(callback_started.wait(), timeout=2)
            task.cancel()
            if repeat_cancellation:
                await asyncio.sleep(0)
                task.cancel()
            outcome: BaseException | None = None
            try:
                await asyncio.wait_for(task, timeout=1)
            except BaseException as exc:
                outcome = exc
        await asyncio.sleep(0)
        leaked = [
            repr(pending.get_coro())
            for pending in asyncio.all_tasks()
            if pending is not asyncio.current_task()
            and not pending.done()
            and any(
                name in repr(pending.get_coro()) for name in ("finish", "invoke_callback", "_cancel_task_repeatedly")
            )
        ]
        return outcome, leaked

    outcome, leaked = asyncio.run(exercise())

    assert isinstance(outcome, asyncio.CancelledError)
    assert callback_calls == 1
    assert leaked == []
    assert not environment._active_processes


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@pytest.mark.parametrize("repeat_cancellation", [False, True])
def test_cancellation_during_timeout_callback_cleanup_is_not_swallowed(
    tmp_path: Path,
    repeat_cancellation: bool,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_environment

    environment = _initialized_local_environment(tmp_path)
    monkeypatch.setattr(local_environment, "_REAP_TERM_SECONDS", 0.01)
    monkeypatch.setattr(local_environment, "_REAP_KILL_SECONDS", 0.01)
    monkeypatch.setattr(local_environment, "_REAP_CANCEL_SECONDS", 0.05)
    callback_cleanup_started = asyncio.Event()

    async def cleanup_awaiting_callback(_text: str, _stream: str) -> None:
        try:
            await asyncio.Event().wait()
        except asyncio.CancelledError:
            callback_cleanup_started.set()
            await asyncio.Event().wait()

    async def exercise() -> tuple[BaseException | object, list[str]]:
        await environment.start()
        with environment.scoped_output_callback(cleanup_awaiting_callback):
            task = asyncio.create_task(environment.exec("sleep 30", timeout_sec=0.02))
            # This is set by the timeout finalizer's own first cancellation,
            # proving caller cancellation lands in its no-exception cleanup
            # path rather than the preceding bounded wait.
            await asyncio.wait_for(callback_cleanup_started.wait(), timeout=2)
            task.cancel()
            if repeat_cancellation:
                await asyncio.sleep(0)
                task.cancel()
            try:
                outcome: BaseException | object = await asyncio.wait_for(task, timeout=1)
            except BaseException as exc:
                outcome = exc
        await asyncio.sleep(0)
        leaked = [
            repr(pending.get_coro())
            for pending in asyncio.all_tasks()
            if pending is not asyncio.current_task()
            and not pending.done()
            and any(
                name in repr(pending.get_coro()) for name in ("finish", "invoke_callback", "_cancel_task_repeatedly")
            )
        ]
        return outcome, leaked

    outcome, leaked = asyncio.run(exercise())

    assert isinstance(outcome, asyncio.CancelledError)
    assert leaked == []
    assert not environment._active_processes


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_large_dense_short_secret_has_the_same_completed_outcome_with_callback(tmp_path: Path) -> None:
    environment = _initialized_local_environment(tmp_path)
    script = "import os; os.write(1, b'x' * 2_000_000)"
    command = f"{shlex.quote(sys.executable)} -c {shlex.quote(script)}"

    async def on_output(_text: str, _stream: str) -> None:
        return None

    async def exercise(with_callback: bool) -> object:
        await environment.start()
        callback_scope = environment.scoped_output_callback(on_output) if with_callback else contextlib.nullcontext()
        with callback_scope:
            return await environment.exec(command, env={"API_KEY": "x"}, timeout_sec=1.5)

    without_callback = asyncio.run(exercise(False))
    with_callback = asyncio.run(exercise(True))

    assert without_callback.return_code == with_callback.return_code == 0
    assert without_callback.stdout == with_callback.stdout
    assert "x" not in (with_callback.stdout or "")


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_streamed_exec_stays_bounded_when_output_closes_before_process_exit(tmp_path: Path) -> None:
    environment = _initialized_local_environment(tmp_path)
    callback_chunks: list[tuple[str, str]] = []

    async def on_output(text: str, stream: str) -> None:
        callback_chunks.append((text, stream))

    async def exercise() -> tuple[object, float]:
        await environment.start()
        started = time.monotonic()
        with environment.scoped_output_callback(on_output):
            result = await environment.exec("exec 1>&- 2>&-; sleep 30", timeout_sec=1)
        return result, time.monotonic() - started

    result, elapsed = asyncio.run(exercise())

    assert elapsed < 3
    assert result.return_code == 124
    assert result.stdout == ""
    assert result.stderr == "Timed out"
    assert callback_chunks == [("Timed out", "stderr")]


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_exec_cancellation_terminates_background_descendants(tmp_path: Path) -> None:
    environment = _local_environment(tmp_path)
    child_ready = environment._workspace / "cancel-child-ready"
    child_pid_path = environment._workspace / "cancel-child-pid"
    marker = environment._workspace / "cancel-child-survived"
    command = (
        # An interrupted sleep must not itself count as a surviving child.
        "(printf ready > cancel-child-ready; sleep 30 && "
        "printf survived > cancel-child-survived) & "
        # Existence signals readiness, so publish only the complete PID.
        "printf '%s' \"$!\" > cancel-child-pid.tmp && "
        "mv cancel-child-pid.tmp cancel-child-pid; wait"
    )
    environment._local_command_guardrail_reason = lambda *_args: ""  # type: ignore[method-assign]

    def process_exists(pid: int) -> bool:
        try:
            os.kill(pid, 0)
        except ProcessLookupError:
            return False
        except PermissionError:
            return True
        return True

    async def run_cancelled() -> None:
        task = asyncio.create_task(environment.exec(command))
        child_pid: int | None = None
        for _ in range(500):
            if child_ready.exists() and child_pid_path.exists():
                child_pid = int(child_pid_path.read_text(encoding="ascii"))
                break
            if task.done():
                pytest.fail(f"command exited before cancellation: {task.result()}")
            await asyncio.sleep(0.01)
        assert child_pid is not None, "the background descendant did not start before cancellation"

        try:
            task.cancel()
            with pytest.raises(asyncio.CancelledError):
                await asyncio.wait_for(task, timeout=5)
            for _ in range(200):
                if not process_exists(child_pid):
                    break
                await asyncio.sleep(0.01)
            assert not process_exists(child_pid), "the background descendant survived command cancellation"
        finally:
            if process_exists(child_pid):
                with contextlib.suppress(ProcessLookupError):
                    os.kill(child_pid, signal.SIGKILL)

    asyncio.run(run_cancelled())

    assert not marker.exists(), "a background descendant wrote after command cancellation"


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@pytest.mark.parametrize("repeat_cancellation", [False, True])
def test_streamed_exec_cancellation_reaps_descendants(
    tmp_path: Path,
    repeat_cancellation: bool,
) -> None:
    environment = _initialized_local_environment(tmp_path)
    environment._local_command_guardrail_reason = lambda *_args: ""  # type: ignore[method-assign]
    secret = "stream-cancel-secret-value"
    child_pid_path = environment._workspace / "stream-cancel-child-pid"
    callback_started = asyncio.Event()
    callback_chunks: list[str] = []

    async def on_output(text: str, _stream: str) -> None:
        callback_chunks.append(text)
        callback_started.set()

    async def exercise() -> int:
        await environment.start()
        with environment.scoped_output_callback(on_output):
            task = asyncio.create_task(
                environment.exec(
                    # Publish the complete PID before callback/file readiness.
                    "(sleep 30) & printf '%s' \"$!\" > stream-cancel-child-pid.tmp; "
                    "mv stream-cancel-child-pid.tmp stream-cancel-child-pid; "
                    'printf "%s\\n" "$CANCEL_TOKEN"; wait',
                    env={"CANCEL_TOKEN": secret},
                )
            )
            await asyncio.wait_for(callback_started.wait(), timeout=5)
            for _ in range(500):
                if child_pid_path.exists():
                    break
                await asyncio.sleep(0.01)
            assert child_pid_path.exists()
            task.cancel()
            if repeat_cancellation:
                await asyncio.sleep(0)
                task.cancel()
            with pytest.raises(asyncio.CancelledError):
                await asyncio.wait_for(task, timeout=5)
        return int(child_pid_path.read_text(encoding="ascii"))

    child_pid = asyncio.run(exercise())

    assert callback_chunks
    assert secret not in "".join(callback_chunks)
    _assert_process_exits(child_pid)


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@pytest.mark.parametrize("repeat_cancellation", [False, True])
def test_streamed_exec_cancellation_during_final_flush_reaps_descendants(
    tmp_path: Path,
    repeat_cancellation: bool,
) -> None:
    environment = _initialized_local_environment(tmp_path)
    environment._local_command_guardrail_reason = lambda *_args: ""  # type: ignore[method-assign]
    child_pid_path = environment._workspace / "flush-cancel-child-pid"
    callback_started = asyncio.Event()
    command = (
        "(trap '' TERM; exec >/dev/null 2>&1; sleep 30) & "
        f"printf '%s' \"$!\" > {shlex.quote(child_pid_path.name)}; "
        # A lone known-token prefix stays buffered until final redactor flush.
        "printf s"
    )

    async def blocked_callback(_text: str, _stream: str) -> None:
        callback_started.set()
        await asyncio.Event().wait()

    async def exercise() -> int:
        await environment.start()
        with environment.scoped_output_callback(blocked_callback):
            task = asyncio.create_task(environment.exec(command))
            await asyncio.wait_for(callback_started.wait(), timeout=5)
            assert child_pid_path.exists()
            assert not environment._active_processes
            child_pid = int(child_pid_path.read_text(encoding="ascii"))
            # The final callback is still blocked, so containment came first.
            _assert_process_exits(child_pid)
            task.cancel()
            if repeat_cancellation:
                await asyncio.sleep(0)
                task.cancel()
            with pytest.raises(asyncio.CancelledError):
                await asyncio.wait_for(task, timeout=5)
        return int(child_pid_path.read_text(encoding="ascii"))

    child_pid: int | None = None
    try:
        child_pid = asyncio.run(exercise())
        _assert_process_exits(child_pid)
    finally:
        if child_pid is None and child_pid_path.exists():
            child_pid = int(child_pid_path.read_text(encoding="ascii"))
        if child_pid is not None:
            with contextlib.suppress(ProcessLookupError):
                os.kill(child_pid, signal.SIGKILL)


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@pytest.mark.parametrize("repeat_cancellation", [False, True])
def test_cancellation_during_timeout_reap_preserves_cancellation_and_containment(
    tmp_path: Path,
    repeat_cancellation: bool,
) -> None:
    environment = _initialized_local_environment(tmp_path)
    environment._local_command_guardrail_reason = lambda *_args: ""  # type: ignore[method-assign]
    child_pid_path = environment._workspace / "timeout-reap-cancel-child-pid"
    child_pid_tmp_path = child_pid_path.with_name(f"{child_pid_path.name}.tmp")
    reap_entered = asyncio.Event()
    original_terminate = environment._terminate_process_tree
    command = (
        "(trap '' TERM; exec >/dev/null 2>&1; sleep 30) & "
        # Existence signals readiness, so publish only the complete PID.
        f"printf '%s' \"$!\" > {shlex.quote(child_pid_tmp_path.name)} && "
        f"mv {shlex.quote(child_pid_tmp_path.name)} {shlex.quote(child_pid_path.name)}; wait"
    )

    async def observed_terminate(
        proc: asyncio.subprocess.Process,
        communication: asyncio.Task[tuple[bytes, bytes]] | None = None,
    ) -> tuple[bytes, bytes]:
        # The exec timeout can expire before a loaded host has started the
        # descendant. Hold the reap until its PID is published, so that the
        # cancellation always interrupts the reap of a live descendant.
        for _ in range(400):
            if child_pid_path.exists():
                break
            await asyncio.sleep(0.01)
        reap_entered.set()
        return await original_terminate(proc, communication)

    async def on_output(_text: str, _stream: str) -> None:
        return None

    async def exercise() -> tuple[BaseException | None, int, bool]:
        await environment.start()
        environment._terminate_process_tree = observed_terminate  # type: ignore[method-assign]
        with environment.scoped_output_callback(on_output):
            task = asyncio.create_task(environment.exec(command, timeout_sec=0.5))
            await asyncio.wait_for(reap_entered.wait(), timeout=5)
            assert child_pid_path.exists()
            task.cancel()
            if repeat_cancellation:
                await asyncio.sleep(0)
                task.cancel()
            outcome: BaseException | None = None
            try:
                await asyncio.wait_for(task, timeout=5)
            except BaseException as exc:
                outcome = exc
        retained_after_exec = bool(environment._active_processes)
        if retained_after_exec:
            await environment.stop(delete=False)
        return outcome, int(child_pid_path.read_text(encoding="ascii")), retained_after_exec

    child_pid: int | None = None
    try:
        outcome, child_pid, retained = asyncio.run(exercise())
        assert isinstance(outcome, asyncio.CancelledError)
        assert not retained
        _assert_process_exits(child_pid)
    finally:
        if child_pid is None and child_pid_path.exists():
            child_pid = int(child_pid_path.read_text(encoding="ascii"))
        if child_pid is not None:
            with contextlib.suppress(ProcessLookupError):
                os.kill(child_pid, signal.SIGKILL)


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@pytest.mark.parametrize("repeat_cancellation", [False, True])
def test_cancellation_remains_primary_when_timeout_reap_fails(
    tmp_path: Path,
    repeat_cancellation: bool,
) -> None:
    from skillevaluator.tier3.harbor import local_environment

    environment = _initialized_local_environment(tmp_path)
    original_terminate = environment._terminate_process_tree
    reap_entered = asyncio.Event()
    release_failure = asyncio.Event()
    secret = "timeout-reap-cleanup-secret"

    async def failing_terminate(
        _proc: asyncio.subprocess.Process,
        _communication: asyncio.Task[tuple[bytes, bytes]] | None = None,
    ) -> tuple[bytes, bytes]:
        async def fail_after_release() -> tuple[bytes, bytes]:
            await release_failure.wait()
            raise PermissionError(errno.EACCES, "Denied", secret)

        cleanup = asyncio.create_task(fail_after_release())
        reap_entered.set()
        return await local_environment._await_task_uninterruptibly(cleanup)

    async def on_output(_text: str, _stream: str) -> None:
        return None

    async def exercise() -> tuple[BaseException | None, bool]:
        await environment.start()
        environment._terminate_process_tree = failing_terminate  # type: ignore[method-assign]
        with environment.scoped_output_callback(on_output):
            task = asyncio.create_task(
                environment.exec(
                    "sleep 30",
                    env={"API_KEY": secret},
                    timeout_sec=0.02,
                )
            )
            await asyncio.wait_for(reap_entered.wait(), timeout=2)
            task.cancel()
            if repeat_cancellation:
                await asyncio.sleep(0)
                task.cancel()
            release_failure.set()
            outcome: BaseException | None = None
            try:
                await asyncio.wait_for(task, timeout=1)
            except BaseException as exc:
                outcome = exc
        retained = bool(environment._active_processes)
        environment._terminate_process_tree = original_terminate  # type: ignore[method-assign]
        await environment.stop(delete=False)
        return outcome, retained

    outcome, retained = asyncio.run(exercise())

    assert isinstance(outcome, asyncio.CancelledError)
    assert isinstance(outcome.__cause__, RuntimeError)
    assert outcome.__cause__.__context__ is None
    assert retained
    assert secret not in str(outcome.__cause__)
    assert secret not in "".join(traceback.format_exception(outcome))
    assert not environment._active_processes


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@pytest.mark.parametrize("lifecycle", ["timeout", "cancel", "repeat-cancel"])
def test_streamed_exec_escalates_after_launcher_exits_with_term_ignoring_descendant(
    tmp_path: Path,
    lifecycle: str,
) -> None:
    environment = _initialized_local_environment(tmp_path)
    environment._local_command_guardrail_reason = lambda *_args: ""  # type: ignore[method-assign]
    child_pid_path = environment._workspace / f"term-ignoring-{lifecycle}.pid"
    child_pid_tmp_path = child_pid_path.with_name(f"{child_pid_path.name}.tmp")
    command = (
        "(trap '' TERM; exec >/dev/null 2>&1; sleep 30) & "
        # Existence signals readiness, so publish only the complete PID.
        f"printf '%s' \"$!\" > {shlex.quote(child_pid_tmp_path.name)} && "
        f"mv {shlex.quote(child_pid_tmp_path.name)} {shlex.quote(child_pid_path.name)}; "
        "printf ready; wait"
    )

    async def on_output(_text: str, _stream: str) -> None:
        return None

    async def exercise() -> object | None:
        await environment.start()
        with environment.scoped_output_callback(on_output):
            # Leave enough startup margin for loaded CI hosts to launch the
            # stdin bootstrap and create the descendant before timeout cleanup.
            task = asyncio.create_task(environment.exec(command, timeout_sec=1.0 if lifecycle == "timeout" else None))
            for _ in range(500):
                if child_pid_path.exists():
                    break
                await asyncio.sleep(0.01)
            assert child_pid_path.exists()
            if lifecycle == "timeout":
                result = await asyncio.wait_for(task, timeout=5)
                assert result.return_code == 124
                return result
            task.cancel()
            if lifecycle == "repeat-cancel":
                await asyncio.sleep(0)
                task.cancel()
            with pytest.raises(asyncio.CancelledError):
                await asyncio.wait_for(task, timeout=5)
            return None

    child_pid: int | None = None
    try:
        asyncio.run(exercise())
        child_pid = int(child_pid_path.read_text(encoding="ascii"))
        # On macOS the group probe can report EPERM after SIGKILL while this
        # descendant is still finishing its exit. Allow bounded exit completion.
        deadline = time.monotonic() + 5
        while True:
            try:
                os.kill(child_pid, 0)
            except ProcessLookupError:
                break
            assert time.monotonic() < deadline, "a SIGTERM-ignoring descendant survived cleanup"
            time.sleep(0.01)
    finally:
        if child_pid is None and child_pid_path.exists():
            child_pid = int(child_pid_path.read_text(encoding="ascii"))
        if child_pid is not None:
            try:
                os.kill(child_pid, 0)
            except ProcessLookupError:
                pass
            else:
                with contextlib.suppress(ProcessLookupError):
                    os.kill(child_pid, signal.SIGKILL)


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_process_tree_cleanup_is_bounded_and_escalates(monkeypatch: pytest.MonkeyPatch) -> None:
    from skillevaluator.tier3.harbor import local_environment

    signals: list[signal.Signals] = []
    monkeypatch.setattr(local_environment, "_REAP_TERM_SECONDS", 0.01)
    monkeypatch.setattr(local_environment, "_REAP_KILL_SECONDS", 0.01)
    monkeypatch.setattr(local_environment.os, "killpg", lambda _pid, value: signals.append(value))

    class FakeProcess:
        pid = 4242
        returncode = None

    async def run_cleanup() -> tuple[bytes, bytes]:
        communication: asyncio.Future[tuple[bytes, bytes]] = asyncio.get_running_loop().create_future()
        return await asyncio.wait_for(
            SkillEvaluatorLocalEnvironment._terminate_process_tree(FakeProcess(), communication),  # type: ignore[arg-type]
            timeout=0.2,
        )

    assert asyncio.run(run_cleanup()) == (b"", b"")
    assert signals == [signal.SIGTERM, signal.SIGKILL]


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_process_tree_cleanup_suppresses_permission_race_after_leader_exit(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    class FakeProcess:
        pid = 4242
        returncode = None

    async def communication() -> tuple[bytes, bytes]:
        return b"stdout", b"stderr"

    monkeypatch.setattr(
        os,
        "killpg",
        lambda *_args: (_ for _ in ()).throw(PermissionError("leader exited")),
    )
    monkeypatch.setattr(
        os,
        "getpgid",
        lambda _pid: (_ for _ in ()).throw(ProcessLookupError()),
    )

    async def exercise() -> tuple[bytes, bytes]:
        task = asyncio.create_task(communication())
        return await SkillEvaluatorLocalEnvironment._terminate_process_tree(  # type: ignore[arg-type]
            FakeProcess(),
            task,
        )

    result = asyncio.run(exercise())

    assert result == (b"stdout", b"stderr")


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_process_tree_cleanup_propagates_live_permission_failure(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    class FakeProcess:
        pid = 4242
        returncode = None

    async def communication() -> tuple[bytes, bytes]:
        await asyncio.Event().wait()
        return b"", b""

    monkeypatch.setattr(
        os,
        "killpg",
        lambda *_args: (_ for _ in ()).throw(PermissionError("live process denied")),
    )
    monkeypatch.setattr(os, "getpgid", lambda pid: pid)

    async def exercise() -> None:
        task = asyncio.create_task(communication())
        try:
            with pytest.raises(PermissionError, match="live process denied"):
                await SkillEvaluatorLocalEnvironment._terminate_process_tree(  # type: ignore[arg-type]
                    FakeProcess(),
                    task,
                )
        finally:
            task.cancel()
            with contextlib.suppress(asyncio.CancelledError):
                await task

    asyncio.run(exercise())


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_process_tree_cleanup_stays_bounded_when_communication_ignores_cancellation(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_environment

    monkeypatch.setattr(local_environment, "_REAP_TERM_SECONDS", 0.01)
    monkeypatch.setattr(local_environment, "_REAP_KILL_SECONDS", 0.01)
    monkeypatch.setattr(local_environment, "_REAP_CANCEL_SECONDS", 0.01, raising=False)
    monkeypatch.setattr(local_environment.os, "killpg", lambda *_args: None)

    async def run_cleanup() -> tuple[bool, bool]:
        started = asyncio.Event()

        async def stubborn_communication() -> tuple[bytes, bytes]:
            started.set()
            try:
                await asyncio.Event().wait()
            except asyncio.CancelledError:
                await asyncio.Event().wait()
            return b"", b""

        communication = asyncio.create_task(stubborn_communication())
        await started.wait()

        class FakeProcess:
            pid = 4242
            returncode = None

        cleanup = asyncio.create_task(
            SkillEvaluatorLocalEnvironment._terminate_process_tree(  # type: ignore[arg-type]
                FakeProcess(),
                communication,
            )
        )
        done, _pending = await asyncio.wait({cleanup}, timeout=0.1)
        finished_within_bound = cleanup in done
        await cleanup
        return finished_within_bound, communication.cancelled()

    # On the manual clock the 0.1s bound counts only the waits that cleanup
    # asked for, not the time a loaded host keeps the event loop descheduled.
    assert asyncio.run(run_cleanup(), loop_factory=_ManualClockEventLoop) == (True, True)


def test_stop_reaps_all_tracked_processes(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
    environment = _local_environment(tmp_path)
    first = object()
    second = object()
    environment._active_processes.update({first: None, second: None})  # type: ignore[dict-item]
    reaped: list[object] = []

    async def reap(proc: object, _communication: object = None) -> tuple[bytes, bytes]:
        reaped.append(proc)
        return b"", b""

    monkeypatch.setattr(environment, "_terminate_process_tree", reap)

    asyncio.run(environment.stop(delete=False))

    assert set(reaped) == {first, second}
    assert environment._active_processes == {}


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_exec_cancellation_during_process_creation_terminates_descendants(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    environment = _local_environment(tmp_path)
    create_subprocess_exec = asyncio.create_subprocess_exec

    async def run_cancelled_during_create() -> None:
        process_created = asyncio.Event()
        release_process = asyncio.Event()
        created: list[asyncio.subprocess.Process] = []

        async def delayed_create(*args: object, **kwargs: object) -> asyncio.subprocess.Process:
            proc = await create_subprocess_exec(*args, **kwargs)
            created.append(proc)
            process_created.set()
            await release_process.wait()
            return proc

        monkeypatch.setattr(asyncio, "create_subprocess_exec", delayed_create)
        task = asyncio.create_task(environment.exec("sleep 30"))
        await asyncio.wait_for(process_created.wait(), timeout=5)
        task.cancel()
        release_process.set()
        try:
            with pytest.raises(asyncio.CancelledError):
                await task
            for _ in range(100):
                if created[0].returncode is not None:
                    break
                await asyncio.sleep(0.01)
            assert created[0].returncode is not None, "spawned launcher survived cancellation during process creation"
        finally:
            if created and created[0].returncode is None:
                with contextlib.suppress(ProcessLookupError):
                    os.killpg(created[0].pid, signal.SIGKILL)
                await created[0].communicate()

    asyncio.run(run_cancelled_during_create())


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_exec_cancellation_does_not_wait_forever_for_uncooperative_process_creation(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_environment

    environment = _local_environment(tmp_path)
    create_subprocess_exec = asyncio.create_subprocess_exec
    monkeypatch.setattr(local_environment, "_CREATION_CANCEL_SECONDS", 0.02, raising=False)

    async def exercise() -> tuple[bool, asyncio.subprocess.Process]:
        process_created = asyncio.Event()
        release_creation = asyncio.Event()
        created: list[asyncio.subprocess.Process] = []

        async def uncooperative_create(*args: object, **kwargs: object) -> asyncio.subprocess.Process:
            process = await create_subprocess_exec(*args, **kwargs)
            created.append(process)
            process_created.set()
            try:
                await release_creation.wait()
            except asyncio.CancelledError:
                await release_creation.wait()
            return process

        monkeypatch.setattr(asyncio, "create_subprocess_exec", uncooperative_create)
        task = asyncio.create_task(environment.exec("sleep 30"))
        await asyncio.wait_for(process_created.wait(), timeout=5)
        task.cancel()
        await asyncio.sleep(0)
        task.cancel()
        done, _pending = await asyncio.wait({task}, timeout=0.2)
        returned_within_bound = task in done
        release_creation.set()
        if task not in done:
            with pytest.raises(asyncio.CancelledError):
                await task
        else:
            with pytest.raises(asyncio.CancelledError):
                task.result()
        for _ in range(500):
            if created[0].returncode is not None:
                break
            await asyncio.sleep(0.01)
        return returned_within_bound, created[0]

    returned_within_bound, process = asyncio.run(exercise())

    assert returned_within_bound
    assert process.returncode is not None


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_stop_fails_closed_until_withheld_process_creation_is_reaped(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_environment

    environment = _local_environment(tmp_path)
    create_subprocess_exec = asyncio.create_subprocess_exec
    monkeypatch.setattr(local_environment, "_CREATION_CANCEL_SECONDS", 0.02)
    created: list[asyncio.subprocess.Process] = []

    async def exercise() -> tuple[asyncio.subprocess.Process, asyncio.subprocess.Process, bool, bool]:
        process_created = asyncio.Event()
        release_creation = asyncio.Event()

        active_process = await create_subprocess_exec(
            "bash",
            "-c",
            "sleep 30",
            stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            start_new_session=True,
        )
        active_communication = asyncio.create_task(active_process.communicate())
        environment._active_processes[active_process] = active_communication

        async def withheld_create(*args: object, **kwargs: object) -> asyncio.subprocess.Process:
            process = await create_subprocess_exec(*args, **kwargs)
            created.append(process)
            process_created.set()
            await release_creation.wait()
            return process

        monkeypatch.setattr(asyncio, "create_subprocess_exec", withheld_create)
        exec_task = asyncio.create_task(environment.exec("sleep 30"))
        await asyncio.wait_for(process_created.wait(), timeout=5)
        exec_task.cancel()
        await asyncio.sleep(0)
        exec_task.cancel()
        with pytest.raises(asyncio.CancelledError):
            await asyncio.wait_for(exec_task, timeout=0.2)

        with pytest.raises(RuntimeError, match="could not confirm process creation containment"):
            await asyncio.wait_for(environment.stop(delete=True), timeout=0.2)
        root_preserved_while_unresolved = environment._root.exists()
        process_alive_while_stop_failed = created[0].returncode is None
        assert active_process.returncode is not None
        assert active_process not in environment._active_processes

        release_creation.set()
        for _ in range(500):
            if created[0].returncode is not None and not environment._creation_cleanups:
                break
            await asyncio.sleep(0.01)
        await environment.stop(delete=True)
        return created[0], active_process, root_preserved_while_unresolved, process_alive_while_stop_failed

    process: asyncio.subprocess.Process | None = None
    active_process: asyncio.subprocess.Process | None = None
    try:
        process, active_process, root_preserved, process_was_alive = asyncio.run(exercise())
        assert root_preserved
        assert process_was_alive
        assert process.returncode is not None
        assert not environment._root.exists()
        assert not environment._pending_creations
        assert not environment._creation_cleanups
    finally:
        leaked_process = process or (created[0] if created else None)
        for process_to_reap in (leaked_process, active_process):
            if process_to_reap is not None and process_to_reap.returncode is None:
                with contextlib.suppress(ProcessLookupError):
                    os.killpg(process_to_reap.pid, signal.SIGKILL)


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@pytest.mark.parametrize("secret", ["PermissionError", "Local process creation cleanup failed"])
def test_failed_late_creation_cleanup_retains_process_for_stop_retry(
    tmp_path: Path,
    caplog: pytest.LogCaptureFixture,
    secret: str,
) -> None:
    environment = _local_environment(tmp_path)
    original_terminate = environment._terminate_process_tree
    process: asyncio.subprocess.Process | None = None
    caplog.set_level(logging.ERROR, logger="asyncio")

    async def exercise() -> tuple[bool, bool, bool]:
        nonlocal process
        process = await asyncio.create_subprocess_exec(
            "bash",
            "-c",
            "sleep 30",
            stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            start_new_session=True,
        )
        creation = asyncio.create_task(asyncio.sleep(0, result=process))
        environment._pending_creations.add(creation)
        cleanup_attempts = 0

        async def fail_twice_then_reap(
            proc: asyncio.subprocess.Process,
            communication: asyncio.Task[tuple[bytes, bytes]] | None = None,
        ) -> tuple[bytes, bytes]:
            nonlocal cleanup_attempts
            cleanup_attempts += 1
            if cleanup_attempts <= 2:
                raise PermissionError(errno.EACCES, "Denied", secret)
            return await original_terminate(proc, communication)

        environment._terminate_process_tree = fail_twice_then_reap  # type: ignore[method-assign]
        cleanup = environment._schedule_creation_cleanup(creation, secret_values={secret})
        with pytest.raises(PermissionError):
            await cleanup
        await asyncio.sleep(0)
        retained_after_late_failure = process in environment._active_processes

        with pytest.raises(RuntimeError, match="cleanup is still pending"):
            await environment.start()
        with pytest.raises(RuntimeError, match="could not confirm process creation containment") as caught:
            await environment.stop(delete=True)
        first_stop_was_redacted = secret not in str(caught.value) and environment._root.exists()

        await environment.stop(delete=True)
        return retained_after_late_failure, first_stop_was_redacted, process.returncode is not None

    try:
        retained, first_stop_redacted, reaped = asyncio.run(exercise())
        assert retained
        assert first_stop_redacted
        assert secret not in caplog.text
        assert reaped
        assert not environment._active_processes
        assert not environment._creation_cleanup_errors
        assert not environment._root.exists()
    finally:
        if process is not None and process.returncode is None:
            with contextlib.suppress(ProcessLookupError):
                os.killpg(process.pid, signal.SIGKILL)


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_creation_completing_during_stop_redacts_cleanup_failure(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    environment = _local_environment(tmp_path)
    create_subprocess_exec = asyncio.create_subprocess_exec
    original_terminate = environment._terminate_process_tree
    secret = "stop-race-cleanup-secret"

    async def stop_after_create(*args: object, **kwargs: object) -> asyncio.subprocess.Process:
        process = await create_subprocess_exec(*args, **kwargs)
        environment._stop_requested = True
        return process

    async def fail_before_reap(
        _proc: asyncio.subprocess.Process,
        _communication: asyncio.Task[tuple[bytes, bytes]] | None = None,
    ) -> tuple[bytes, bytes]:
        raise PermissionError(errno.EACCES, "Denied", secret)

    async def exercise() -> tuple[BaseException, bool]:
        monkeypatch.setattr(asyncio, "create_subprocess_exec", stop_after_create)
        environment._terminate_process_tree = fail_before_reap  # type: ignore[method-assign]
        with pytest.raises(RuntimeError) as caught:
            await environment.exec("sleep 30", env={"API_KEY": secret})
        retained = bool(environment._active_processes)
        environment._terminate_process_tree = original_terminate  # type: ignore[method-assign]
        await environment.stop(delete=False)
        return caught.value, retained

    caught, retained = asyncio.run(exercise())

    assert retained
    assert caught.__context__ is None
    assert secret not in str(caught)
    assert secret not in "".join(traceback.format_exception(caught))
    assert not environment._active_processes


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_repeated_cancellation_during_process_creation_still_reaps_launcher(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    environment = _local_environment(tmp_path)
    create_subprocess_exec = asyncio.create_subprocess_exec

    async def run_repeated_cancellation() -> None:
        process_created = asyncio.Event()
        release_process = asyncio.Event()
        created: list[asyncio.subprocess.Process] = []

        async def delayed_create(*args: object, **kwargs: object) -> asyncio.subprocess.Process:
            proc = await create_subprocess_exec(*args, **kwargs)
            created.append(proc)
            process_created.set()
            await release_process.wait()
            return proc

        monkeypatch.setattr(asyncio, "create_subprocess_exec", delayed_create)
        task = asyncio.create_task(environment.exec("sleep 30"))
        await asyncio.wait_for(process_created.wait(), timeout=5)
        task.cancel()
        # Let exec enter its cancellation handler and start waiting for the
        # still-running creation task before delivering a second cancellation.
        await asyncio.sleep(0)
        task.cancel()
        release_process.set()
        try:
            with pytest.raises(asyncio.CancelledError):
                await asyncio.wait_for(task, timeout=5)
            for _ in range(100):
                if created[0].returncode is not None:
                    break
                await asyncio.sleep(0.01)
            assert created[0].returncode is not None, "repeated cancellation orphaned the spawned launcher"
        finally:
            if created and created[0].returncode is None:
                with contextlib.suppress(ProcessLookupError):
                    os.killpg(created[0].pid, signal.SIGKILL)
                await created[0].communicate()

    asyncio.run(run_repeated_cancellation())


@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_process_tree_cleanup_finishes_after_repeated_cancellation(monkeypatch: pytest.MonkeyPatch) -> None:
    from skillevaluator.tier3.harbor import local_environment

    monkeypatch.setattr(local_environment, "_REAP_TERM_SECONDS", 0.01)
    monkeypatch.setattr(local_environment, "_REAP_KILL_SECONDS", 0.01)

    async def run_cleanup() -> list[signal.Signals]:
        signals: list[signal.Signals] = []
        term_sent = asyncio.Event()
        communication: asyncio.Future[tuple[bytes, bytes]] = asyncio.get_running_loop().create_future()

        def killpg(_pid: int, value: signal.Signals) -> None:
            signals.append(value)
            if value == signal.SIGTERM:
                term_sent.set()
            elif not communication.done():
                communication.set_result((b"", b""))

        monkeypatch.setattr(local_environment.os, "killpg", killpg)

        class FakeProcess:
            pid = 4242
            returncode = None

        task = asyncio.create_task(
            SkillEvaluatorLocalEnvironment._terminate_process_tree(FakeProcess(), communication)  # type: ignore[arg-type]
        )
        await asyncio.wait_for(term_sent.wait(), timeout=1)
        task.cancel()
        await asyncio.sleep(0)
        task.cancel()
        with pytest.raises(asyncio.CancelledError):
            await asyncio.wait_for(task, timeout=1)
        return signals

    assert asyncio.run(run_cleanup()) == [signal.SIGTERM, signal.SIGKILL]


@pytest.mark.parametrize(
    "name",
    [
        "LD_PRELOAD",
        "LD_AUDIT",
        "LD_LIBRARY_PATH",
        "DYLD_INSERT_LIBRARIES",
        "PYTHONPATH",
        "PYTHONHOME",
        "BASH_ENV",
        "ENV",
        "ZDOTDIR",
        "RUBYOPT",
        "PERL5OPT",
        "NODE_OPTIONS",
    ],
)
@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_runtime_injection_env_is_blocked_before_launcher(name: str, tmp_path: Path) -> None:
    environment = _local_environment(tmp_path)

    async def unexpected_launch(*_args: object, **_kwargs: object) -> None:
        pytest.fail("loader-controlled task environment reached the host launcher")

    with pytest.MonkeyPatch.context() as patch:
        patch.setattr(asyncio, "create_subprocess_exec", unexpected_launch)
        result = asyncio.run(
            environment.exec("true", env={name: "attacker-controlled", "HARBOR_INSTRUCTION": "task prompt"})
        )

    assert result.return_code == 126
    assert name in (result.stderr or "")


@pytest.mark.parametrize("name", ["BASH_ENV", "ENV", "NODE_OPTIONS", "PYTHONPATH"])
@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_empty_evaluator_loader_reset_is_removed_before_launcher(name: str, tmp_path: Path) -> None:
    environment = _initialized_local_environment(tmp_path)

    async def exercise() -> object:
        await environment.start()
        try:
            return await environment.exec(f'test -z "${{{name}+x}}"', env={name: ""})
        finally:
            await environment.stop(delete=True)

    result = asyncio.run(exercise())

    assert result.return_code == 0, result.stderr


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@pytest.mark.parametrize(
    ("command", "cwd", "env", "secret"),
    [
        ("true", "/tmp/ordinary-secret-path", {"API_KEY": "/tmp/ordinary-secret-path"}, "/tmp/ordinary-secret-path"),
        ("true", "/tmp/sk-Ab1Cd2Ef3Gh4Ij5Kl6Mn7Op8", None, "sk-Ab1Cd2Ef3Gh4Ij5Kl6Mn7Op8"),
        ('touch "$SECRET_PATH"', None, {"SECRET_PATH": "/tmp/guardrail-secret-path"}, "/tmp/guardrail-secret-path"),
        ("rm -rf /", None, {"API_KEY": "Local mode command blocked"}, "Local mode command blocked"),
        ('touch "$API_KEY"', None, {"API_KEY": "/x"}, "/x"),
    ],
)
def test_prelaunch_diagnostics_are_streamed_and_redacted_once(
    tmp_path: Path,
    caplog: pytest.LogCaptureFixture,
    command: str,
    cwd: str | None,
    env: dict[str, str] | None,
    secret: str,
) -> None:
    environment = _initialized_local_environment(tmp_path)
    callback_chunks: list[tuple[str, str]] = []
    caplog.set_level(logging.WARNING)

    async def on_output(text: str, stream: str) -> None:
        callback_chunks.append((text, stream))

    async def exercise() -> object:
        await environment.start()
        with environment.scoped_output_callback(on_output):
            return await environment.exec(command, cwd=cwd, env=env)

    result = asyncio.run(exercise())
    callback_stderr = "".join(text for text, stream in callback_chunks if stream == "stderr")

    assert result.return_code == 126
    assert callback_stderr == result.stderr
    assert secret not in (result.stderr or "")
    assert secret not in callback_stderr
    assert secret not in caplog.text


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@pytest.mark.parametrize(
    ("name", "value", "is_sensitive"),
    [
        ("MONKEY", "banana", False),
        ("KEYBOARD", "clacky", False),
        ("MYSECRET", "legacy-secret-value", True),
        # The long-value legacy fallback remains intentionally conservative.
        ("MONKEY", "bananabanana", True),
        ("API_KEY", "x", True),
    ],
)
def test_output_redaction_balances_component_names_and_legacy_fallback(
    tmp_path: Path,
    name: str,
    value: str,
    is_sensitive: bool,
) -> None:
    environment = _initialized_local_environment(tmp_path)
    callback_chunks: list[str] = []

    async def on_output(text: str, _stream: str) -> None:
        callback_chunks.append(text)

    async def exercise() -> object:
        await environment.start()
        with environment.scoped_output_callback(on_output):
            return await environment.exec(f'printf %s "${name}"', env={name: value})

    result = asyncio.run(exercise())
    callback_output = "".join(callback_chunks)

    assert callback_output == result.stdout
    assert (value not in callback_output) is is_sensitive
    if not is_sensitive:
        assert callback_output == value


@pytest.mark.parametrize("name", ["BASH_ENV", "ENV", "LD_PRELOAD", "PYTHONPATH"])
def test_empty_runtime_injection_env_is_dropped(name: str) -> None:
    assert SkillEvaluatorLocalEnvironment._filter_command_env({name: ""}, protected=set()) == {}


def test_unblocked_empty_runtime_env_is_preserved() -> None:
    assert SkillEvaluatorLocalEnvironment._filter_command_env({"EXAMPLE": ""}, protected=set()) == {"EXAMPLE": ""}


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_persistent_runtime_injection_env_is_blocked_before_launcher(tmp_path: Path) -> None:
    environment = _local_environment(tmp_path, persistent_env={"NODE_OPTIONS": "--require=/tmp/attack.js"})

    result = asyncio.run(environment.exec("true"))

    assert result.return_code == 126
    assert "NODE_OPTIONS" in (result.stderr or "")


def test_validate_local_agents_rejects_unsupported() -> None:
    assert validate_local_agents(["opencode", "gemini-cli", "aider"]) == ["aider", "gemini-cli"]
    assert validate_local_agents(["claude-code", "codex", "opencode"]) == []


def test_ensure_local_runtimes_reports_vendor_hint_when_missing(monkeypatch: pytest.MonkeyPatch) -> None:
    # Force the CLI to look absent (it's installed on dev hosts) and confirm the
    # hint points to a vendor-supported install command.
    from skillevaluator.tier3.harbor import local_runtime

    monkeypatch.setattr(local_runtime, "find_runtime_command", lambda _agent, **_: None)
    errors = ensure_local_runtimes(["opencode"])
    assert len(errors) == 1
    assert "opencode" in errors[0]
    assert "npm install" in errors[0] or "brew install" in errors[0]


def test_ensure_local_runtimes_rejects_installed_cli_with_failing_version(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    monkeypatch.setattr(local_runtime, "find_runtime_command", lambda _agent, **_: "/fake/codex")
    monkeypatch.setattr(
        local_runtime,
        "subprocess",
        SimpleNamespace(
            run=lambda *args, **_kwargs: subprocess.CompletedProcess(args[0], 17, stdout="", stderr="broken")
        ),
        raising=False,
    )

    errors = ensure_local_runtimes(["codex"])

    assert len(errors) == 1
    assert "codex" in errors[0]
    assert "/fake/codex" in errors[0]
    assert "--version" in errors[0]
    assert "exit 17" in errors[0]
    assert "npm install" in errors[0] or "brew install" in errors[0]


def test_ensure_local_runtimes_rejects_installed_cli_when_version_times_out(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    monkeypatch.setattr(local_runtime, "find_runtime_command", lambda _agent, **_: "/fake/opencode")

    def time_out(*args, **kwargs):
        raise subprocess.TimeoutExpired(args[0], timeout=kwargs["timeout"])

    monkeypatch.setattr(local_runtime, "subprocess", SimpleNamespace(run=time_out), raising=False)

    errors = ensure_local_runtimes(["opencode"])

    assert len(errors) == 1
    assert "opencode" in errors[0]
    assert "--version" in errors[0]
    assert "timed out" in errors[0]
    assert "npm install" in errors[0] or "brew install" in errors[0]


def test_ensure_local_runtimes_reports_sandbox_wrap_failure(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime, local_sandbox

    command = tmp_path / "bin" / "opencode"
    command.parent.mkdir()
    command.write_text("binary", encoding="utf-8")
    command.chmod(0o755)
    monkeypatch.setattr(local_runtime, "find_runtime_command", lambda _agent, **_: str(command))

    class RejectingSandbox:
        def wrap(self, *_args, **_kwargs):
            raise local_sandbox.SandboxUnavailable("cannot determine existing host HOME roots")

    errors = ensure_local_runtimes(["opencode"], sandbox=RejectingSandbox())

    assert len(errors) == 1
    assert "opencode" in errors[0]
    assert "sandboxed --version" in errors[0]
    assert "cannot determine existing host HOME roots" in errors[0]
    assert "--env-mode docker" in errors[0]


def test_ensure_local_runtimes_probes_with_effective_strict_read_policy(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    command = tmp_path / "bin" / "opencode"
    command.parent.mkdir()
    command.write_text("binary", encoding="utf-8")
    command.chmod(0o755)
    monkeypatch.setattr(local_runtime, "find_runtime_command", lambda _agent, **_: str(command))
    monkeypatch.setattr(local_runtime, "runtime_command_roots", lambda *_args, **_kwargs: [command])
    monkeypatch.setattr(
        local_runtime,
        "subprocess",
        SimpleNamespace(run=lambda argv, **_kwargs: subprocess.CompletedProcess(argv, 0, stdout="1.0", stderr="")),
        raising=False,
    )
    captured: dict[str, object] = {}

    class CaptureSandbox:
        def wrap(self, argv, **kwargs):
            captured.update(kwargs)
            return argv

    assert ensure_local_runtimes(["opencode"], sandbox=CaptureSandbox(), strict_reads=True) == []
    assert captured["strict_reads"] is True


def test_ensure_local_runtimes_reports_disappearing_runtime_root(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    command = tmp_path / "bin" / "opencode"
    command.parent.mkdir()
    command.write_text("binary", encoding="utf-8")
    command.chmod(0o755)
    monkeypatch.setattr(local_runtime, "find_runtime_command", lambda _agent, **_: str(command))

    def disappearing_roots(*_args, **_kwargs):
        raise FileNotFoundError("selected runtime disappeared before sandbox preparation")

    monkeypatch.setattr(local_runtime, "runtime_command_roots", disappearing_roots)

    class UnusedSandbox:
        def wrap(self, *_args, **_kwargs):
            raise AssertionError("wrap must not run after read-root discovery fails")

    errors = ensure_local_runtimes(["opencode"], sandbox=UnusedSandbox())

    assert len(errors) == 1
    assert "sandboxed --version" in errors[0]
    assert "FileNotFoundError" in errors[0]
    assert "selected runtime disappeared" in errors[0]
    assert "--env-mode docker" in errors[0]


def test_ensure_local_runtimes_reports_runtime_command_symlink_cycle(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    command = tmp_path / "bin" / "opencode"
    command.parent.mkdir()
    loop_target = command.parent / "opencode-loop"
    command.symlink_to(loop_target.name)
    loop_target.symlink_to(command.name)
    monkeypatch.setattr(local_runtime, "find_runtime_command", lambda _agent, **_: str(command))

    class UnusedSandbox:
        def wrap(self, *_args, **_kwargs):
            raise AssertionError("wrap must not run after runtime path resolution fails")

    errors = ensure_local_runtimes(["opencode"], sandbox=UnusedSandbox())

    assert len(errors) == 1
    assert "sandboxed --version" in errors[0]
    assert "RuntimePathResolutionError" in errors[0]
    assert "symlink" in errors[0].lower()
    assert "--env-mode docker" in errors[0]


def test_ensure_local_runtimes_reports_shebang_interpreter_symlink_cycle(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    interpreter = tmp_path / "interpreters" / "python3"
    interpreter.parent.mkdir()
    loop_target = interpreter.parent / "python3-loop"
    interpreter.symlink_to(loop_target.name)
    loop_target.symlink_to(interpreter.name)
    command = tmp_path / "bin" / "opencode"
    command.parent.mkdir()
    command.write_text(f"#!{interpreter}\n", encoding="utf-8")
    command.chmod(0o755)
    monkeypatch.setattr(local_runtime, "find_runtime_command", lambda _agent, **_: str(command))

    class UnusedSandbox:
        def wrap(self, *_args, **_kwargs):
            raise AssertionError("wrap must not run after interpreter path resolution fails")

    errors = ensure_local_runtimes(["opencode"], sandbox=UnusedSandbox())

    assert len(errors) == 1
    assert "sandboxed --version" in errors[0]
    assert "RuntimePathResolutionError" in errors[0]
    assert str(interpreter) in errors[0]
    assert "--env-mode docker" in errors[0]


@pytest.mark.parametrize(
    "error_type",
    [TypeError, AssertionError, RuntimeError, ValueError, OSError, subprocess.SubprocessError],
)
def test_ensure_local_runtimes_preserves_sandbox_programming_errors(
    error_type: type[Exception],
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    command = tmp_path / "bin" / "opencode"
    command.parent.mkdir()
    command.write_text("binary", encoding="utf-8")
    command.chmod(0o755)
    monkeypatch.setattr(local_runtime, "find_runtime_command", lambda _agent, **_: str(command))

    class BrokenSandbox:
        def wrap(self, *_args, **_kwargs):
            raise error_type("programming defect")

    with pytest.raises(error_type, match="programming defect"):
        ensure_local_runtimes(["opencode"], sandbox=BrokenSandbox())


@pytest.mark.parametrize("signal_type", [KeyboardInterrupt, GeneratorExit])
def test_ensure_local_runtimes_preserves_sandbox_control_flow_signals(
    signal_type: type[BaseException],
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    command = tmp_path / "bin" / "opencode"
    command.parent.mkdir()
    command.write_text("binary", encoding="utf-8")
    command.chmod(0o755)
    monkeypatch.setattr(local_runtime, "find_runtime_command", lambda _agent, **_: str(command))

    class InterruptingSandbox:
        def wrap(self, *_args, **_kwargs):
            raise signal_type()

    with pytest.raises(signal_type):
        ensure_local_runtimes(["opencode"], sandbox=InterruptingSandbox())


def test_ensure_local_runtimes_accepts_working_version_with_safe_environment(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    captured: dict[str, object] = {}
    monkeypatch.setenv("OPENAI_API_KEY", "must-not-reach-probe")
    monkeypatch.setattr(local_runtime, "find_runtime_command", lambda _agent, **_: "/fake/opencode")

    def capture_run(argv, **kwargs):
        captured.update({"argv": argv, **kwargs})
        probe_env = kwargs["env"]
        assert "OPENAI_API_KEY" not in probe_env
        assert Path(probe_env["HOME"]).is_dir()
        assert Path(probe_env["TMPDIR"]).is_dir()
        assert Path(kwargs["cwd"]).is_relative_to(Path(probe_env["HOME"]).parent)
        return subprocess.CompletedProcess(argv, 0, stdout="opencode 1.2.3\n", stderr="")

    monkeypatch.setattr(local_runtime, "subprocess", SimpleNamespace(run=capture_run), raising=False)

    assert ensure_local_runtimes(["opencode"], env={"PATH": "/custom/bin", "API_TOKEN": "secret"}) == []
    assert captured["argv"] == ["/fake/opencode", "--version"]
    assert isinstance(captured["timeout"], int | float) and captured["timeout"] > 0


def test_local_runtime_install_hint_has_no_internal_url() -> None:
    from skillevaluator.tier3.harbor.local_runtime import local_runtime_install_command

    hint = local_runtime_install_command(["claude-code", "codex", "opencode"])
    assert "npm install" in hint or "brew install" in hint


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_runtime_read_paths_include_symlinked_npm_package_without_sibling_bin(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    npm_root = tmp_path / ".npm-global"
    command = npm_root / "lib" / "node_modules" / "opencode-ai" / "bin" / "opencode"
    command.parent.mkdir(parents=True)
    command.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
    command.chmod(0o755)
    bin_dir = npm_root / "bin"
    bin_dir.mkdir()
    (bin_dir / "opencode").symlink_to(command)
    monkeypatch.setenv("PATH", str(bin_dir))

    roots = local_runtime.runtime_command_roots(["opencode"], runtime_root=tmp_path / "managed")

    assert (bin_dir / "opencode").absolute() in roots
    assert command.parent.parent.resolve() in roots
    assert bin_dir.resolve() not in roots
    assert npm_root.resolve() not in roots


def test_runtime_read_paths_keep_direct_node_modules_bin_shim_exact(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    bin_dir = tmp_path / "node_modules" / ".bin"
    bin_dir.mkdir(parents=True)
    command = bin_dir / "opencode"
    command.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
    command.chmod(0o755)
    sibling = bin_dir / "unselected-agent"
    sibling.write_text("must remain hidden", encoding="utf-8")
    monkeypatch.setenv("PATH", str(bin_dir))

    roots = local_runtime.runtime_command_roots(["opencode"], runtime_root=tmp_path / "managed")

    assert command.absolute() in roots
    assert bin_dir.resolve() not in roots
    assert sibling.resolve() not in roots


def test_runtime_read_paths_keep_direct_binary_exact(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    command = tmp_path / "tools" / "opencode"
    command.parent.mkdir()
    command.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
    command.chmod(0o755)
    monkeypatch.setenv("PATH", str(command.parent))

    roots = local_runtime.runtime_command_roots(["opencode"], runtime_root=tmp_path / "managed")

    shell_paths = {Path("/bin").resolve() / "sh", Path("/bin/sh").resolve()}
    assert roots[0] == command.absolute()
    assert set(roots[1:]) == shell_paths
    assert command.parent.resolve() not in roots


def test_runtime_read_paths_keep_single_file_symlink_exact(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    target = tmp_path / "tools" / "opencode"
    target.parent.mkdir()
    target.write_text("binary", encoding="utf-8")
    target.chmod(0o755)
    bin_dir = tmp_path / ".local" / "bin"
    bin_dir.mkdir(parents=True)
    command = bin_dir / "opencode"
    command.symlink_to(target)
    monkeypatch.setenv("PATH", str(bin_dir))

    roots = local_runtime.runtime_command_roots(["opencode"], runtime_root=tmp_path / "managed")

    assert command.absolute() in roots
    assert target.resolve() in roots
    assert tmp_path.resolve() not in roots
    assert bin_dir.resolve() not in roots
    assert target.parent.resolve() not in roots


def test_runtime_read_paths_preserve_claude_standalone_layout(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    target = tmp_path / ".local" / "share" / "claude" / "versions" / "2.1.198"
    target.parent.mkdir(parents=True)
    target.write_text("binary", encoding="utf-8")
    target.chmod(0o755)
    bin_dir = tmp_path / ".local" / "bin"
    bin_dir.mkdir(parents=True)
    command = bin_dir / "claude"
    command.symlink_to(target)
    monkeypatch.setenv("PATH", str(bin_dir))

    roots = local_runtime.runtime_command_roots(["claude-code"], runtime_root=tmp_path / "managed")

    assert roots == [command.absolute(), target.resolve()]
    assert (tmp_path / ".local" / "share" / "claude").resolve() not in roots


def test_runtime_read_paths_resolve_env_shebang_helper_without_sibling_directory(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    command = tmp_path / "agent-bin" / "opencode"
    command.parent.mkdir()
    command.write_text("#!/usr/bin/env helper\n", encoding="utf-8")
    command.chmod(0o755)
    helper = tmp_path / "helper-bin" / "helper"
    helper.parent.mkdir()
    helper.write_text("binary", encoding="utf-8")
    helper.chmod(0o755)
    monkeypatch.setenv("PATH", os.pathsep.join((str(command.parent), str(helper.parent), os.defpath)))

    roots = local_runtime.runtime_command_roots(["opencode"], runtime_root=tmp_path / "managed")

    assert command.absolute() in roots
    assert Path("/usr/bin/env") in roots
    assert helper.absolute() in roots
    assert command.parent.resolve() not in roots
    assert helper.parent.resolve() not in roots


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_runtime_read_paths_include_selected_homebrew_dependency_kegs_and_shipped_config(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    prefix = tmp_path / "homebrew"
    node_keg = prefix / "Cellar" / "node" / "25.0.0"
    node = node_keg / "bin" / "node"
    node.parent.mkdir(parents=True)
    node.write_text("binary", encoding="utf-8")
    node.chmod(0o755)
    (node_keg / "INSTALL_RECEIPT.json").write_text(
        json.dumps(
            {
                "runtime_dependencies": [
                    {"full_name": "openssl@3", "pkg_version": "3.0.0"},
                ]
            }
        ),
        encoding="utf-8",
    )

    openssl_keg = prefix / "Cellar" / "openssl@3" / "3.0.0"
    bottled_config = openssl_keg / ".bottle" / "etc" / "openssl@3" / "openssl.cnf"
    bottled_config.parent.mkdir(parents=True)
    bottled_config.write_text("shipped default", encoding="utf-8")
    openssl_opt = prefix / "opt" / "openssl@3"
    openssl_opt.parent.mkdir(parents=True)
    openssl_opt.symlink_to(openssl_keg)
    installed_config = prefix / "etc" / "openssl@3" / "openssl.cnf"
    installed_config.parent.mkdir(parents=True)
    installed_config.write_text("installed default", encoding="utf-8")
    shipped_link = bottled_config.parent / "outside.cnf"
    shipped_link.write_text("shipped default", encoding="utf-8")
    outside_target = tmp_path / "outside-homebrew.txt"
    outside_target.write_text("must remain hidden", encoding="utf-8")
    installed_link = installed_config.parent / "outside.cnf"
    installed_link.symlink_to(outside_target)
    shipped_prefix_link = bottled_config.parent / "prefix-secret.cnf"
    shipped_prefix_link.write_text("shipped default", encoding="utf-8")
    prefix_secret = prefix / "var" / "private-service" / "secret"
    prefix_secret.parent.mkdir(parents=True)
    prefix_secret.write_text("must remain hidden", encoding="utf-8")
    installed_prefix_link = installed_config.parent / "prefix-secret.cnf"
    installed_prefix_link.symlink_to(prefix_secret)
    shipped_etc_link = bottled_config.parent / "other-service.cnf"
    shipped_etc_link.write_text("shipped default", encoding="utf-8")
    other_service_secret = prefix / "etc" / "other-service" / "private.cnf"
    other_service_secret.parent.mkdir(parents=True)
    other_service_secret.write_text("must remain hidden", encoding="utf-8")
    installed_etc_link = installed_config.parent / "other-service.cnf"
    installed_etc_link.symlink_to(other_service_secret)
    unrelated_secret = prefix / "etc" / "openssl@3" / "private" / "host.key"
    unrelated_secret.parent.mkdir()
    unrelated_secret.write_text("must remain hidden", encoding="utf-8")

    helper_bin = prefix / "bin"
    helper_bin.mkdir()
    (helper_bin / "node").symlink_to(node)
    agent_bin = tmp_path / "agent-bin"
    agent_bin.mkdir()
    command = agent_bin / "codex"
    command.write_text("#!/usr/bin/env node\n", encoding="utf-8")
    command.chmod(0o755)
    monkeypatch.setenv("PATH", os.pathsep.join((str(agent_bin), str(helper_bin), os.defpath)))

    roots = local_runtime.runtime_command_roots(["codex"], runtime_root=tmp_path / "managed")

    assert node_keg.resolve() in roots
    assert openssl_opt.absolute() in roots
    assert openssl_keg.resolve() in roots
    assert installed_config.resolve() in roots
    assert installed_link.absolute() not in roots
    assert outside_target.resolve() not in roots
    assert installed_prefix_link.absolute() not in roots
    assert prefix_secret.resolve() not in roots
    assert installed_etc_link.absolute() not in roots
    assert other_service_secret.resolve() not in roots
    assert unrelated_secret.resolve() not in roots
    assert installed_config.parent.resolve() not in roots
    assert (prefix / "Cellar").resolve() not in roots


def test_homebrew_runtime_roots_ignore_malformed_or_cyclic_receipt_dependencies(tmp_path: Path) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    prefix = tmp_path / "homebrew"
    keg = prefix / "Cellar" / "node" / "25.0.0"
    binary = keg / "bin" / "node"
    binary.parent.mkdir(parents=True)
    binary.write_text("binary", encoding="utf-8")
    (keg / "INSTALL_RECEIPT.json").write_text(
        json.dumps(
            {
                "runtime_dependencies": [
                    {"full_name": "bad\u0000name", "pkg_version": "1"},
                    {"full_name": "cycle", "pkg_version": "1"},
                    {"full_name": "escape", "pkg_version": "1"},
                ]
            }
        ),
        encoding="utf-8",
    )
    cycle = prefix / "opt" / "cycle"
    cycle.parent.mkdir(parents=True)
    cycle.symlink_to(cycle)
    outside_formula = tmp_path / "outside-formula"
    outside_keg = outside_formula / "1"
    outside_keg.mkdir(parents=True)
    cellar_formula = prefix / "Cellar" / "escape"
    cellar_formula.symlink_to(outside_formula)
    (prefix / "opt" / "escape").symlink_to(outside_keg)

    assert local_runtime._homebrew_runtime_roots(binary) == [keg.resolve()]


def test_homebrew_shipped_config_paths_reject_symlinked_layout_roots(tmp_path: Path) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    prefix = tmp_path / "homebrew"
    keg = prefix / "Cellar" / "node" / "25.0.0"
    shipped = keg / ".bottle" / "etc" / "node" / "runtime.conf"
    shipped.parent.mkdir(parents=True)
    shipped.write_text("default", encoding="utf-8")
    outside_etc = tmp_path / "outside-etc"
    installed = outside_etc / "node" / "runtime.conf"
    installed.parent.mkdir(parents=True)
    installed.write_text("secret", encoding="utf-8")
    (prefix / "etc").symlink_to(outside_etc)

    assert local_runtime._homebrew_shipped_config_paths(prefix, keg) == []

    other_prefix = tmp_path / "other-homebrew"
    other_keg = other_prefix / "Cellar" / "node" / "25.0.0"
    outside_bottle = tmp_path / "outside-bottle"
    outside_shipped = outside_bottle / "node" / "runtime.conf"
    outside_shipped.parent.mkdir(parents=True)
    outside_shipped.write_text("default", encoding="utf-8")
    (other_keg / ".bottle").mkdir(parents=True)
    (other_keg / ".bottle" / "etc").symlink_to(outside_bottle)
    other_installed = other_prefix / "etc" / "node" / "runtime.conf"
    other_installed.parent.mkdir(parents=True)
    other_installed.write_text("secret", encoding="utf-8")

    assert local_runtime._homebrew_shipped_config_paths(other_prefix, other_keg) == []


def test_find_runtime_command_does_not_search_other_managed_agent_bins(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    collision = tmp_path / "runtimes" / "opencode" / "bin" / "codex"
    collision.parent.mkdir(parents=True)
    collision.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
    collision.chmod(0o755)
    monkeypatch.setenv("PATH", os.defpath)

    assert local_runtime.find_runtime_command("codex", runtime_root=tmp_path / "runtimes") is None


def test_version_probe_executes_canonical_parent_when_path_directory_is_symlinked(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    actual_bin = tmp_path / "actual-bin"
    actual_bin.mkdir()
    command = actual_bin / "opencode"
    command.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
    command.chmod(0o755)
    path_alias = tmp_path / "path-alias"
    path_alias.symlink_to(actual_bin, target_is_directory=True)
    monkeypatch.setenv("PATH", str(path_alias))
    captured: dict[str, object] = {}

    def capture_run(argv, **_kwargs):
        captured["argv"] = argv
        return subprocess.CompletedProcess(argv, 0, stdout="opencode 1.0\n", stderr="")

    monkeypatch.setattr(local_runtime, "subprocess", SimpleNamespace(run=capture_run))

    assert ensure_local_runtimes(["opencode"], runtime_root=tmp_path / "managed") == []
    assert captured["argv"] == [str(command), "--version"]


def test_local_subprocess_env_excludes_sibling_managed_agent_bins(tmp_path: Path) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    runtime_root = tmp_path / "runtimes"
    codex_bin = runtime_root / "codex" / "bin"
    opencode_bin = runtime_root / "opencode" / "bin"
    codex_bin.mkdir(parents=True)
    opencode_bin.mkdir(parents=True)
    inherited = os.pathsep.join((str(opencode_bin), str(codex_bin), os.defpath))

    env = local_runtime.local_subprocess_env(
        runtime_root=runtime_root,
        runtime_agents=["codex"],
        base_env={"PATH": inherited},
    )
    path_parts = env["PATH"].split(os.pathsep)

    assert str(codex_bin) in path_parts
    assert str(opencode_bin) not in path_parts


def test_runtime_path_excludes_symlink_alias_to_sibling_managed_bin(tmp_path: Path) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    runtime_root = tmp_path / "runtimes"
    selected_bin = runtime_root / "codex" / "bin"
    sibling_bin = runtime_root / "opencode" / "bin"
    selected_bin.mkdir(parents=True)
    sibling_bin.mkdir(parents=True)
    collision = sibling_bin / "codex"
    collision.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
    collision.chmod(0o755)
    sibling_alias = tmp_path / "sibling-alias"
    sibling_alias.symlink_to(sibling_bin, target_is_directory=True)
    host_bin = tmp_path / "host-bin"
    host_bin.mkdir()

    path = local_runtime.runtime_path(
        runtime_root,
        os.pathsep.join((str(sibling_alias), str(host_bin))),
        agents=["codex"],
    )
    path_parts = path.split(os.pathsep)

    assert path_parts[0] == str(selected_bin)
    assert str(sibling_alias) not in path_parts
    assert str(host_bin) in path_parts
    assert local_runtime.shutil.which("codex", path=path) is None


def test_runtime_path_canonicalizes_selected_bin_below_symlinked_runtime_root(tmp_path: Path) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    real_runtime_root = tmp_path / "real-runtimes"
    selected_bin = real_runtime_root / "codex" / "bin"
    selected_bin.mkdir(parents=True)
    command = selected_bin / "codex"
    command.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
    command.chmod(0o755)
    runtime_alias = tmp_path / "runtime-alias"
    runtime_alias.symlink_to(real_runtime_root, target_is_directory=True)

    path = local_runtime.runtime_path(runtime_alias, "", agents=["codex"])
    path_parts = path.split(os.pathsep)

    assert path_parts[0] == str(selected_bin.resolve())
    assert str(runtime_alias / "codex" / "bin") not in path_parts
    assert local_runtime.shutil.which("codex", path=path) == str(command.resolve())


def test_runtime_path_canonicalizes_normal_host_bin_alias_and_preserves_unresolved(tmp_path: Path) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    host_bin = tmp_path / "versions" / "current" / "bin"
    host_bin.mkdir(parents=True)
    helper = host_bin / "helper"
    helper.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
    helper.chmod(0o755)
    host_alias = tmp_path / "host-bin-alias"
    host_alias.symlink_to(host_bin, target_is_directory=True)
    unresolved = tmp_path / "not-installed-yet"

    path = local_runtime.runtime_path(
        tmp_path / "managed",
        os.pathsep.join((str(host_alias), str(host_bin), str(unresolved), str(host_alias))),
        agents=["codex"],
    )
    path_parts = path.split(os.pathsep)

    assert path_parts == [str(host_bin.resolve()), str(unresolved)]
    assert local_runtime.shutil.which("helper", path=path) == str(helper.resolve())


def test_runtime_path_drops_non_absolute_inherited_components(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime

    workspace = tmp_path / "workspace"
    relative_bin = workspace / "evilbin"
    relative_bin.mkdir(parents=True)
    absolute_bin = tmp_path / "absolute-bin"
    absolute_bin.mkdir()
    absolute_file = tmp_path / "not-a-bin-directory"
    absolute_file.write_text("not a PATH directory", encoding="utf-8")
    unresolved_absolute = tmp_path / "not-installed-yet"
    monkeypatch.chdir(workspace)

    path = local_runtime.runtime_path(
        tmp_path / "managed",
        os.pathsep.join(
            (
                str(absolute_bin),
                "evilbin",
                ".",
                "..",
                "$HOME/bin",
                "relative/missing",
                "",
                str(absolute_file),
                str(unresolved_absolute),
            )
        ),
        agents=["codex"],
    )

    assert path.split(os.pathsep) == [str(absolute_bin.resolve()), str(unresolved_absolute)]


@pytest.mark.parametrize("runtime_agent", [None, "aider"])
def test_local_environment_rejects_missing_or_unsupported_runtime_agent(
    monkeypatch: pytest.MonkeyPatch,
    runtime_agent: str | None,
) -> None:
    monkeypatch.setattr(BaseEnvironment, "__init__", lambda *_args, **_kwargs: None)

    with pytest.raises(ValueError, match="runtime_agent"):
        SkillEvaluatorLocalEnvironment(runtime_agent=runtime_agent)


def test_local_environment_requires_runtime_agent_keyword(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(BaseEnvironment, "__init__", lambda *_args, **_kwargs: None)

    with pytest.raises(TypeError, match="runtime_agent"):
        SkillEvaluatorLocalEnvironment()


def test_runtime_ro_binds_include_only_selected_agent(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_environment

    runtime_root = tmp_path / "runtimes"
    selected_root = runtime_root / "codex"
    sibling_root = runtime_root / "opencode"
    selected_root.mkdir(parents=True)
    sibling_root.mkdir()
    captured: list[tuple[str, ...]] = []

    def capture_runtime_roots(agents, **_kwargs):
        captured.append(tuple(agents))
        return []

    monkeypatch.setattr(local_environment, "runtime_command_roots", capture_runtime_roots)
    environment = object.__new__(SkillEvaluatorLocalEnvironment)
    environment._runtime_root = runtime_root
    environment._runtime_agent = "codex"

    binds = environment._runtime_ro_binds()

    assert captured == [("codex",)]
    assert selected_root.resolve() not in binds
    assert sibling_root.resolve() not in binds
    assert runtime_root.resolve() not in binds


def test_runtime_ro_binds_preserve_exact_selected_agent_symlink(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_environment

    target = tmp_path / "tools" / "opencode"
    target.parent.mkdir()
    target.write_text("binary", encoding="utf-8")
    target.chmod(0o755)
    command = tmp_path / ".local" / "bin" / "opencode"
    command.parent.mkdir(parents=True)
    command.symlink_to(target)
    monkeypatch.setattr(
        local_environment,
        "runtime_command_roots",
        lambda *_args, **_kwargs: [command.absolute(), target.resolve()],
    )
    environment = object.__new__(SkillEvaluatorLocalEnvironment)
    environment._runtime_root = tmp_path / "managed"
    environment._runtime_agent = "opencode"

    binds = environment._runtime_ro_binds()

    assert command.absolute() in binds
    assert target.resolve() in binds
    assert command.parent.resolve() not in binds


def test_strict_runtime_ro_binds_use_exact_interpreter_and_python_library_roots(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    import sysconfig

    from skillevaluator.tier3.harbor import local_environment

    target = tmp_path / "python-install" / "bin" / "python3.12"
    target.parent.mkdir(parents=True)
    target.write_text("binary", encoding="utf-8")
    interpreter = tmp_path / "venv" / "bin" / "python"
    interpreter.parent.mkdir(parents=True)
    interpreter.symlink_to(target)
    library_roots = {
        name: tmp_path / "python-install" / "lib" / name for name in ("stdlib", "platstdlib", "purelib", "platlib")
    }
    for path in library_roots.values():
        path.mkdir(parents=True)

    monkeypatch.setattr(sys, "executable", str(interpreter))
    monkeypatch.setattr(sys, "prefix", str(tmp_path / "venv"))
    monkeypatch.setattr(sys, "exec_prefix", "/usr/local")
    monkeypatch.setattr(sys, "base_prefix", "/opt")
    monkeypatch.setattr(sys, "base_exec_prefix", "/usr")
    monkeypatch.setattr(sysconfig, "get_paths", lambda: {name: str(path) for name, path in library_roots.items()})
    monkeypatch.setattr(sysconfig, "get_config_var", lambda _name: None)
    monkeypatch.setattr(local_environment, "runtime_command_roots", lambda *_args, **_kwargs: [])
    environment = object.__new__(SkillEvaluatorLocalEnvironment)
    environment._runtime_root = tmp_path / "managed"
    environment._runtime_agent = "opencode"
    environment._strict_reads = True

    binds = environment._runtime_ro_binds()

    assert interpreter.absolute() in binds
    assert target.resolve() in binds
    assert set(library_roots.values()).issubset(binds)
    assert interpreter.parent.resolve() not in binds
    assert Path(sys.prefix).resolve() not in binds
    assert Path(sys.exec_prefix).resolve() not in binds
    assert Path(sys.base_prefix).resolve() not in binds
    assert Path(sys.base_exec_prefix).resolve() not in binds


def test_non_strict_runtime_ro_binds_keep_prefix_and_interpreter_parent(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_environment

    prefix = tmp_path / "venv"
    interpreter = prefix / "bin" / "python"
    interpreter.parent.mkdir(parents=True)
    interpreter.write_text("binary", encoding="utf-8")
    monkeypatch.setattr(sys, "executable", str(interpreter))
    monkeypatch.setattr(sys, "prefix", str(prefix))
    monkeypatch.setattr(sys, "exec_prefix", str(prefix))
    monkeypatch.setattr(sys, "base_prefix", str(prefix))
    monkeypatch.setattr(sys, "base_exec_prefix", str(prefix))
    monkeypatch.setattr(local_environment, "runtime_command_roots", lambda *_args, **_kwargs: [])
    environment = object.__new__(SkillEvaluatorLocalEnvironment)
    environment._runtime_root = tmp_path / "managed"
    environment._runtime_agent = "opencode"
    environment._strict_reads = False

    binds = environment._runtime_ro_binds()

    assert prefix.resolve() in binds
    assert interpreter.parent.resolve() in binds


def test_strict_exec_filters_broad_prefixes_and_publishes_selected_npm_runtime(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    import sysconfig

    from skillevaluator.tier3.harbor import local_runtime

    package = tmp_path / "host" / "usr" / "local" / "lib" / "node_modules" / "opencode-ai"
    target = package / "bin" / "opencode"
    target.parent.mkdir(parents=True)
    target.write_text("binary", encoding="utf-8")
    command = tmp_path / "host" / "usr" / "local" / "bin" / "opencode"
    command.parent.mkdir(parents=True)
    command.symlink_to(Path("../lib/node_modules/opencode-ai/bin/opencode"))
    narrow_site_packages = tmp_path / "host" / "opt" / "venv" / "lib" / "python" / "site-packages"
    narrow_site_packages.mkdir(parents=True)

    monkeypatch.setattr(local_runtime, "find_runtime_command", lambda *_args, **_kwargs: str(command))
    monkeypatch.setattr(sys, "prefix", "/usr")
    monkeypatch.setattr(sys, "exec_prefix", "/usr/local")
    monkeypatch.setattr(sys, "base_prefix", "/opt")
    monkeypatch.setattr(sys, "base_exec_prefix", "/tmp")
    monkeypatch.setattr(
        sysconfig,
        "get_paths",
        lambda: {
            "stdlib": str(narrow_site_packages.parent / "stdlib"),
            "platstdlib": str(narrow_site_packages.parent / "platstdlib"),
            "purelib": str(narrow_site_packages),
            "platlib": str(narrow_site_packages),
        },
    )
    monkeypatch.setattr(sysconfig, "get_config_var", lambda _name: None)
    (narrow_site_packages.parent / "stdlib").mkdir()
    (narrow_site_packages.parent / "platstdlib").mkdir()
    monkeypatch.setattr(local_sandbox, "_safe_host_homes", lambda: (Path.home().resolve(),))

    environment = _local_environment(tmp_path)
    environment._strict_reads = True
    captured: dict[str, object] = {}
    real_sandbox = local_sandbox.Sandbox(local_sandbox.SandboxPlan("bubblewrap", "kernel", "capture"))

    class CaptureSandbox:
        plan = real_sandbox.plan

        @staticmethod
        def wrap(argv: list[str], **kwargs: object) -> list[str]:
            captured["extra_ro"] = kwargs["extra_ro"]
            captured["deny_reads"] = kwargs["deny_reads"]
            captured["wrapped"] = real_sandbox.wrap(argv, **kwargs)  # type: ignore[arg-type]
            return argv

    environment._sandbox = CaptureSandbox()

    result = asyncio.run(environment.exec("printf ok"))

    assert result.return_code == 0, result.stderr
    assert result.stdout == "ok"
    wrapped = captured["wrapped"]
    extra_ro = captured["extra_ro"]
    assert isinstance(wrapped, list)
    assert isinstance(extra_ro, list)
    assert captured["deny_reads"] == (Path(os.environ["SKILLEVALUATOR_OUTPUT_PROVENANCE_KEY_FILE"]),)
    ro_binds = {tuple(wrapped[index + 1 : index + 3]) for index, value in enumerate(wrapped) if value == "--ro-bind"}
    symlinks = {tuple(wrapped[index + 1 : index + 3]) for index, value in enumerate(wrapped) if value == "--symlink"}
    broad_roots = {"/", "/usr", "/usr/local", "/opt", "/tmp", "/private/tmp", "/var/tmp", str(Path.home())}
    assert not any(destination in broad_roots for _source, destination in ro_binds | symlinks)
    assert (str(target.resolve()), str(command.absolute())) in symlinks
    assert (str(package.resolve()), str(package.resolve())) in ro_binds
    assert (str(narrow_site_packages.resolve()), str(narrow_site_packages.resolve())) in ro_binds
    assert Path(sys.executable).parent.resolve() not in extra_ro


def test_runtime_ro_binds_do_not_expose_whole_managed_agent_root(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_environment

    managed_root = tmp_path / "managed" / "codex"
    command = managed_root / "bin" / "codex"
    command.parent.mkdir(parents=True)
    command.write_text("binary", encoding="utf-8")
    command.chmod(0o755)
    sibling = managed_root / "sibling-auth.txt"
    sibling.write_text("DENY", encoding="utf-8")
    monkeypatch.setattr(
        local_environment,
        "runtime_command_roots",
        lambda *_args, **_kwargs: [command.resolve()],
    )
    environment = object.__new__(SkillEvaluatorLocalEnvironment)
    environment._runtime_root = tmp_path / "managed"
    environment._runtime_agent = "codex"

    binds = environment._runtime_ro_binds()

    assert command.resolve() in binds
    assert managed_root.resolve() not in binds
    assert sibling.resolve() not in binds


def test_evaluator_python_path_uses_only_selected_runtime_prefix(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_environment

    captured: dict[str, object] = {}

    def capture_bins(runtime_root, *, agents=None):
        captured.update({"runtime_root": runtime_root, "agents": agents})
        return []

    monkeypatch.setattr(local_environment, "runtime_bin_dirs", capture_bins)
    environment = object.__new__(SkillEvaluatorLocalEnvironment)
    environment._runtime_root = tmp_path / "managed"
    environment._runtime_agent = "codex"

    environment._path_with_evaluator_python(os.defpath)

    assert captured["agents"] == ["codex"]


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_shell_path_rewrite_quotes_unquoted_path_with_spaces(tmp_path: Path) -> None:
    target = tmp_path / "result with spaces"
    target.mkdir()
    environment = object.__new__(SkillEvaluatorLocalEnvironment)
    environment._path_map = lambda: [("/logs", target)]

    rewritten = environment._rewrite_command("printf ok > /logs/output.txt")
    result = subprocess.run(["bash", "-c", rewritten], capture_output=True, text=True, timeout=30)

    assert result.returncode == 0, result.stderr
    assert (target / "output.txt").read_text(encoding="utf-8") == "ok"


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_shell_path_rewrite_preserves_existing_quotes(tmp_path: Path) -> None:
    target = tmp_path / "result with spaces"
    target.mkdir()
    environment = object.__new__(SkillEvaluatorLocalEnvironment)
    environment._path_map = lambda: [("/logs", target)]

    rewritten = environment._rewrite_command('printf ok > "/logs/output.txt"')
    result = subprocess.run(["bash", "-c", rewritten], capture_output=True, text=True, timeout=30)

    assert result.returncode == 0, result.stderr
    assert (target / "output.txt").read_text(encoding="utf-8") == "ok"


@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_shell_path_rewrite_handles_exact_path_before_shell_separator(tmp_path: Path) -> None:
    target = tmp_path / "result with spaces"
    target.mkdir()
    environment = object.__new__(SkillEvaluatorLocalEnvironment)
    environment._path_map = lambda: [("/workspace", target)]

    rewritten = environment._rewrite_command("cd /workspace && printf ok > output.txt")
    result = subprocess.run(["bash", "-c", rewritten], capture_output=True, text=True, timeout=30)

    assert result.returncode == 0, result.stderr
    assert (target / "output.txt").read_text(encoding="utf-8") == "ok"


def test_rewrite_env_values_does_not_add_shell_quotes(tmp_path: Path) -> None:
    target = tmp_path / "result with spaces"
    environment = object.__new__(SkillEvaluatorLocalEnvironment)
    environment._path_map = lambda: [("/logs", target)]

    assert environment._rewrite_env_values({"OUTPUT": "/logs/output.txt"}) == {"OUTPUT": str(target / "output.txt")}


def test_local_opencode_confines_project_discovery_to_the_run_workspace(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    agent = object.__new__(SkillEvaluatorLocalOpenCode)
    agent.logs_dir = tmp_path
    agent.model_name = "nvidia/openai/gpt-oss-120b"
    agent.mcp_servers = []
    agent._opencode_config = {}
    agent.render_instruction = lambda instruction: instruction
    agent._build_register_skills_command = lambda: "register-skills"
    agent.build_cli_flags = lambda: ""
    calls: list[tuple[str, dict[str, str]]] = []

    async def capture_exec(
        _environment: object,
        command: str,
        env: dict[str, str] | None = None,
        **_kwargs: object,
    ) -> None:
        calls.append((command, dict(env or {})))

    agent.exec_as_agent = capture_exec
    monkeypatch.setenv("OPENAI_API_KEY", "test-key")
    monkeypatch.setenv("OPENAI_BASE_URL", "https://integrate.api.nvidia.com/v1")

    asyncio.run(agent.run("test instruction", object(), None))

    assert calls[0][0] == "git -C /workspace init -q"
    assert all(env["OPENCODE_TEST_HOME"] == "/workspace" for _, env in calls)
    assert all(env["OPENCODE_DISABLE_PROJECT_CONFIG"] == "1" for _, env in calls)


def test_local_opencode_confinement_is_provider_independent(monkeypatch: pytest.MonkeyPatch) -> None:
    agent = object.__new__(SkillEvaluatorLocalOpenCode)
    agent.model_name = "openai/gpt-4.1-mini"
    calls: list[tuple[str, dict[str, str]]] = []

    async def fake_upstream_run(
        self: OpenCode,
        instruction: str,
        environment: object,
        context: object,
    ) -> None:
        _ = (instruction, context)
        await self.exec_as_agent(environment, command="opencode run", env={"OPENAI_API_KEY": "test-key"})

    async def capture_parent_exec(
        _self: OpenCode,
        _environment: object,
        command: str,
        env: dict[str, str] | None = None,
        **_kwargs: object,
    ) -> None:
        calls.append((command, dict(env or {})))

    monkeypatch.setattr(OpenCode, "run", fake_upstream_run)
    monkeypatch.setattr(OpenCode, "exec_as_agent", capture_parent_exec)

    asyncio.run(agent.run("test instruction", object(), None))

    assert calls[0][0] == "git -C /workspace init -q"
    assert calls[1][0] == "opencode run"
    assert all(env["OPENCODE_FAKE_VCS"] == "git" for _, env in calls)
    assert all(env["OPENCODE_TEST_HOME"] == "/workspace" for _, env in calls)
    assert all(env["OPENCODE_DISABLE_PROJECT_CONFIG"] == "1" for _, env in calls)


def test_doctor_prerequisite_check_receives_selected_agents(monkeypatch: pytest.MonkeyPatch) -> None:
    from skillevaluator.tier3 import commands as tier3_commands

    captured: dict[str, object] = {}
    monkeypatch.setattr(
        tier3_commands,
        "resolve_llm_provider",
        lambda: _provider("openai", api_key="key", base_url="https://api.openai.com/v1"),
    )
    monkeypatch.setattr(
        tier3_commands,
        "_check_prerequisites",
        lambda **kwargs: captured.update(kwargs) or [],
    )

    assert tier3_commands.doctor(agents="opencode", env_mode="local") == 0
    assert captured["agents"] == ["opencode"]


def test_local_prerequisite_probes_runtime_inside_detected_sandbox(monkeypatch: pytest.MonkeyPatch) -> None:
    from skillevaluator.tier3.harbor import local_runtime, local_sandbox

    sandbox = object()
    captured: dict[str, object] = {}
    monkeypatch.setattr(local_sandbox, "detect", lambda _mode: sandbox)
    monkeypatch.setenv("SKILLEVALUATOR_LOCAL_STRICT_READS", "1")
    monkeypatch.setattr(
        local_runtime,
        "ensure_local_runtimes",
        lambda agents, **kwargs: captured.update({"agents": agents, **kwargs}) or [],
    )

    assert _check_prerequisites(env_mode="local", agents=["opencode"]) == []
    assert captured["agents"] == ["opencode"]
    assert captured["sandbox"] is sandbox
    assert captured["strict_reads"] is True


@pytest.mark.parametrize("mode", local_sandbox.SANDBOX_MODES)
def test_local_prerequisite_rejects_native_windows_for_every_sandbox_mode(
    monkeypatch: pytest.MonkeyPatch,
    mode: str,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime, runner

    monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
    monkeypatch.setattr(local_sandbox.platform, "system", lambda: "Windows")
    monkeypatch.setenv(local_sandbox.SANDBOX_MODE_ENV, mode)
    monkeypatch.setattr(
        local_runtime,
        "ensure_local_runtimes",
        lambda *_args, **_kwargs: pytest.fail("native Windows must fail before runtime probing"),
    )

    errors = _check_prerequisites(env_mode="local", agents=["opencode"])

    assert len(errors) == 1
    assert "Native Windows local mode is unsupported" in errors[0]
    assert "WSL2" in errors[0]
    assert "--env-mode docker" in errors[0]


def test_run_harbor_eval_rejects_native_windows_before_provider_or_config(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import runner

    monkeypatch.setattr(local_sandbox.platform, "system", lambda: "Windows")
    monkeypatch.setattr(
        runner,
        "resolve_llm_provider",
        lambda: pytest.fail("native Windows must fail before provider resolution"),
    )
    monkeypatch.setattr(
        runner,
        "load_evals_config",
        lambda _path: pytest.fail("native Windows must fail before config loading"),
    )

    result = runner.run_harbor_eval(tmp_path, ["opencode"], env_mode="local")

    assert result == {
        "error": [
            "Native Windows local mode is unsupported, including with "
            "SKILLEVALUATOR_LOCAL_SANDBOX=prefer or off. "
            "Use WSL2 for Linux local mode or --env-mode docker."
        ]
    }


def test_local_prerequisite_reports_missing_host_home_as_readiness_error(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import local_runtime, local_sandbox, runner

    command = tmp_path / "bin" / "opencode"
    command.parent.mkdir()
    command.write_text("binary", encoding="utf-8")
    command.chmod(0o755)
    sandbox = local_sandbox.Sandbox(local_sandbox.SandboxPlan("seatbelt", "kernel-macos", "test"))
    monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
    monkeypatch.setattr(local_sandbox, "detect", lambda _mode: sandbox)
    monkeypatch.setattr(local_sandbox, "pwd", None)
    monkeypatch.setattr(local_sandbox.platform, "system", lambda: "Darwin")
    monkeypatch.setattr(local_runtime, "find_runtime_command", lambda _agent, **_: str(command))
    monkeypatch.delenv("HOME", raising=False)

    errors = _check_prerequisites(env_mode="local", agents=["opencode"])

    assert len(errors) == 1
    assert "opencode" in errors[0]
    assert "sandboxed --version" in errors[0]
    assert "host HOME" in errors[0]


def test_doctor_uses_local_build_bridge_for_mixed_agents(
    monkeypatch: pytest.MonkeyPatch,
    capsys: pytest.CaptureFixture[str],
) -> None:
    from skillevaluator.tier3 import commands as tier3_commands

    monkeypatch.setattr(
        tier3_commands,
        "resolve_llm_provider",
        lambda: _provider(
            "nv_build",
            api_key="nvidia-build-key",
            base_url="https://integrate.api.nvidia.com/v1",
        ),
    )
    monkeypatch.setattr(tier3_commands, "_check_prerequisites", lambda **_kwargs: [])
    monkeypatch.setenv("OPENAI_API_KEY", "independent-openai-key")
    monkeypatch.delenv("OPENAI_BASE_URL", raising=False)

    assert tier3_commands.doctor(agents="codex,opencode", env_mode="local") == 0
    output = capsys.readouterr().out
    assert "Agent runtime credential" in output
    assert "Codex runtime credential" not in output
    assert "agent container" not in output


def test_doctor_accepts_nvidia_only_credentials_for_local_claude_bridge(
    monkeypatch: pytest.MonkeyPatch,
    capsys: pytest.CaptureFixture[str],
) -> None:
    from skillevaluator.tier3 import commands as tier3_commands

    monkeypatch.setattr(
        tier3_commands,
        "resolve_llm_provider",
        lambda: _provider(
            "nv_build",
            api_key="nvidia-build-key",
            base_url="https://integrate.api.nvidia.com/v1",
        ),
    )
    monkeypatch.setattr(tier3_commands, "_check_prerequisites", lambda **_kwargs: [])
    monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)

    assert tier3_commands.doctor(agents="claude-code", env_mode="local") == 0
    assert "runtime credential" in capsys.readouterr().out


def test_doctor_local_codex_ignores_native_credentials_and_accepts_explicit_build_model(
    monkeypatch: pytest.MonkeyPatch,
    capsys: pytest.CaptureFixture[str],
) -> None:
    from skillevaluator.tier3 import commands as tier3_commands

    monkeypatch.setattr(
        tier3_commands,
        "resolve_llm_provider",
        lambda: _provider(
            "nv_build",
            api_key="nvidia-build-key",
            base_url="https://integrate.api.nvidia.com/v1",
        ),
    )
    monkeypatch.setattr(tier3_commands, "_check_prerequisites", lambda **_kwargs: [])
    monkeypatch.setenv("OPENAI_API_KEY", "independent-openai-key")
    monkeypatch.setenv("OPENAI_BASE_URL", "https://api.openai.com/v1")

    assert (
        tier3_commands.doctor(
            agents="codex",
            env_mode="local",
            agent_model=("codex=nvidia/nemotron-3.5-lightning-30b-a3b",),
        )
        == 0
    )
    output = capsys.readouterr().out
    assert "operator credential and model plan resolved" in " ".join(output.split())


def test_doctor_accepts_isolated_docker_bridge_and_direct_agents(
    monkeypatch: pytest.MonkeyPatch,
    capsys: pytest.CaptureFixture[str],
) -> None:
    from skillevaluator.tier3 import commands as tier3_commands

    monkeypatch.setattr(
        tier3_commands,
        "resolve_llm_provider",
        lambda: _provider(
            "nv_build",
            api_key="nvidia-build-key",
            base_url="https://integrate.api.nvidia.com/v1",
        ),
    )
    monkeypatch.setattr(tier3_commands, "_check_prerequisites", lambda **_kwargs: [])
    monkeypatch.setenv("OPENAI_API_KEY", "independent-openai-key")
    monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
    monkeypatch.delenv("NVIDIA_API_KEY", raising=False)

    assert (
        tier3_commands.doctor(
            agents="codex,opencode",
            env_mode="docker",
            agent_model=("codex=nvidia/nemotron-3.5-lightning-30b-a3b",),
        )
        == 0
    )
    output = capsys.readouterr().out
    assert "Agent runtime credential" in output
    assert "operator credential and model plan resolved" in " ".join(output.split())


def test_harbor_preflight_system_exit_becomes_a_diagnostic(monkeypatch: pytest.MonkeyPatch) -> None:
    from harbor.environments.factory import EnvironmentFactory

    monkeypatch.setattr(
        "skillevaluator.tier3.harbor.runner.subprocess.run",
        lambda *_args, **_kwargs: subprocess.CompletedProcess([], 0, stdout="Docker Compose version v2", stderr=""),
    )
    monkeypatch.setattr(
        EnvironmentFactory,
        "run_preflight",
        lambda *_args, **_kwargs: (_ for _ in ()).throw(SystemExit("  daemon down\n")),
    )

    assert _check_prerequisites(env_mode="docker", agents=[]) == [
        "Harbor environment 'docker' is not ready: daemon down"
    ]


def test_singularity_prerequisite_requires_the_cli_harbor_invokes(monkeypatch: pytest.MonkeyPatch) -> None:
    from skillevaluator.tier3.harbor import runner

    monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
    monkeypatch.setattr(runner.shutil, "which", lambda executable: None if executable == "singularity" else "/bin/tool")

    assert _check_prerequisites(env_mode="singularity", agents=[]) == [
        "Harbor environment 'singularity' requires the singularity CLI on PATH."
    ]


def test_islo_prerequisite_requires_nonempty_host_api_key(monkeypatch: pytest.MonkeyPatch) -> None:
    from skillevaluator.tier3.harbor import runner

    monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
    monkeypatch.setenv("ISLO_API_KEY", "   ")

    assert _check_prerequisites(env_mode="islo", agents=[]) == [
        "Harbor environment 'islo' requires a non-empty ISLO_API_KEY in the host environment."
    ]


def test_modal_custom_config_path_does_not_suppress_harbor_auth_preflight(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    from harbor.environments.factory import EnvironmentFactory

    from skillevaluator.tier3.harbor import runner

    config_path = tmp_path / "modal.toml"
    config_path.write_text("[default]\n", encoding="utf-8")
    monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
    monkeypatch.setattr(runner.importlib.util, "find_spec", lambda _name: object())
    monkeypatch.setenv("MODAL_CONFIG_PATH", str(config_path))
    monkeypatch.setattr(
        EnvironmentFactory,
        "run_preflight",
        lambda *_args, **_kwargs: (_ for _ in ()).throw(
            SystemExit(
                "Modal requires authentication. Run 'modal token new' to set up credentials, "
                "or set MODAL_TOKEN_ID and MODAL_TOKEN_SECRET environment variables."
            )
        ),
    )

    assert _check_prerequisites(env_mode="modal", agents=[]) == [
        "Harbor environment 'modal' is not ready: Modal requires authentication. "
        "Run 'modal token new' to set up credentials, or set MODAL_TOKEN_ID and "
        "MODAL_TOKEN_SECRET environment variables."
    ]


def test_modal_custom_config_path_must_be_an_existing_regular_file(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    from skillevaluator.tier3.harbor import runner

    monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
    monkeypatch.setenv("MODAL_CONFIG_PATH", str(tmp_path / "missing.toml"))

    assert _check_prerequisites(env_mode="modal", agents=[]) == [
        "MODAL_CONFIG_PATH must name an existing regular file."
    ]


def test_modal_custom_config_path_rejects_directory(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    from skillevaluator.tier3.harbor import runner

    monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
    monkeypatch.setenv("MODAL_CONFIG_PATH", str(tmp_path))

    assert _check_prerequisites(env_mode="modal", agents=[]) == [
        "MODAL_CONFIG_PATH must name an existing regular file."
    ]


def test_modal_custom_config_path_stat_error_is_a_bounded_diagnostic(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import runner

    class UnreadablePath:
        def expanduser(self) -> UnreadablePath:
            return self

        def is_file(self) -> bool:
            raise OSError("secret path detail")

    monkeypatch.setenv("MODAL_CONFIG_PATH", "/private/config")
    monkeypatch.setattr(runner, "Path", lambda _value: UnreadablePath())

    assert runner._modal_custom_config_status() == (
        False,
        "MODAL_CONFIG_PATH must name an existing regular file.",
    )


def test_modal_custom_config_path_unknown_user_is_a_bounded_diagnostic(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import runner

    monkeypatch.setenv("MODAL_CONFIG_PATH", "~definitely-no-such-user-issue79/modal.toml")

    assert runner._modal_custom_config_status() == (
        False,
        "MODAL_CONFIG_PATH must name an existing regular file.",
    )


@pytest.mark.parametrize("configured_path", ["", "   "])
def test_modal_custom_config_path_must_not_be_blank(
    monkeypatch: pytest.MonkeyPatch,
    configured_path: str,
) -> None:
    from skillevaluator.tier3.harbor import runner

    monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
    monkeypatch.setenv("MODAL_CONFIG_PATH", configured_path)

    assert _check_prerequisites(env_mode="modal", agents=[]) == [
        "MODAL_CONFIG_PATH must name an existing regular file."
    ]


def test_modal_custom_config_does_not_hide_unrelated_preflight_exit(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    from harbor.environments.factory import EnvironmentFactory

    from skillevaluator.tier3.harbor import runner

    config_path = tmp_path / "modal.toml"
    config_path.write_text("[default]\n", encoding="utf-8")
    monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
    monkeypatch.setattr(runner.importlib.util, "find_spec", lambda _name: object())
    monkeypatch.setenv("MODAL_CONFIG_PATH", str(config_path))
    monkeypatch.setattr(
        EnvironmentFactory,
        "run_preflight",
        lambda *_args, **_kwargs: (_ for _ in ()).throw(SystemExit("Modal daemon unavailable")),
    )

    assert _check_prerequisites(env_mode="modal", agents=[]) == [
        "Harbor environment 'modal' is not ready: Modal daemon unavailable"
    ]


def _install_fake_kubernetes(
    monkeypatch: pytest.MonkeyPatch,
    *,
    kube_config_error: bool = False,
    pod_list_error: Exception | None = None,
) -> dict[str, object]:
    calls: dict[str, object] = {"kube": [], "incluster": 0, "pods": [], "closed": 0}

    class ConfigException(Exception):
        pass

    def load_kube_config(**kwargs: str) -> None:
        cast_calls = calls["kube"]
        assert isinstance(cast_calls, list)
        cast_calls.append(dict(kwargs))
        if kube_config_error:
            raise ConfigException("no kube config")

    def load_incluster_config() -> None:
        calls["incluster"] = int(calls["incluster"]) + 1

    class ApiClient:
        def close(self) -> None:
            calls["closed"] = int(calls["closed"]) + 1

    class CoreV1Api:
        def __init__(self, api_client: ApiClient) -> None:
            assert isinstance(api_client, ApiClient)

        def list_namespaced_pod(self, **kwargs: object) -> None:
            cast_calls = calls["pods"]
            assert isinstance(cast_calls, list)
            cast_calls.append(dict(kwargs))
            if pod_list_error is not None:
                raise pod_list_error

    kubernetes = SimpleNamespace(
        client=SimpleNamespace(ApiClient=ApiClient, CoreV1Api=CoreV1Api),
        config=SimpleNamespace(
            ConfigException=ConfigException,
            load_kube_config=load_kube_config,
            load_incluster_config=load_incluster_config,
        ),
    )
    monkeypatch.setitem(sys.modules, "kubernetes", kubernetes)
    return calls


def test_ack_cluster_readiness_uses_explicit_config_context_and_bounded_namespaced_pod_list(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import runner

    calls = _install_fake_kubernetes(monkeypatch)

    runner._check_ack_cluster_readiness(
        {"namespace": "skill-evals", "context": "production", "kubeconfig": "/config/ack"}
    )

    assert calls == {
        "kube": [{"context": "production", "config_file": "/config/ack"}],
        "incluster": 0,
        "pods": [{"namespace": "skill-evals", "limit": 1, "_request_timeout": (5, 10)}],
        "closed": 1,
    }


def test_ack_cluster_readiness_falls_back_to_incluster_and_closes_on_api_error(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import runner

    calls = _install_fake_kubernetes(
        monkeypatch,
        kube_config_error=True,
        pod_list_error=RuntimeError("namespaced pod list forbidden"),
    )

    with pytest.raises(RuntimeError, match="namespaced pod list forbidden"):
        runner._check_ack_cluster_readiness({"namespace": "skill-evals"})

    assert calls == {
        "kube": [{}],
        "incluster": 1,
        "pods": [{"namespace": "skill-evals", "limit": 1, "_request_timeout": (5, 10)}],
        "closed": 1,
    }


def test_ack_prerequisite_performs_namespaced_pod_list_access_check(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from harbor.environments.factory import EnvironmentFactory

    from skillevaluator.tier3.harbor import runner

    captured: list[tuple[dict[str, object], dict[str, str]]] = []
    subprocess_env = {"PATH": "/usr/bin", "KUBECONFIG": "/config/ack"}
    monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
    monkeypatch.setattr(EnvironmentFactory, "run_preflight", lambda *_args, **_kwargs: None)
    monkeypatch.setattr(
        runner,
        "_check_ack_cluster_readiness_subprocess",
        lambda kwargs, *, subprocess_env: captured.append((dict(kwargs), dict(subprocess_env))),
    )

    assert (
        _check_prerequisites(
            env_mode="ack",
            agents=[],
            environment_kwargs={"namespace": "skill-evals", "context": "production"},
            subprocess_env=subprocess_env,
        )
        == []
    )
    assert captured == [
        (
            {"namespace": "skill-evals", "context": "production"},
            subprocess_env,
        )
    ]


def test_ack_namespace_access_error_is_redacted(monkeypatch: pytest.MonkeyPatch) -> None:
    from harbor.environments.factory import EnvironmentFactory

    from skillevaluator.tier3.harbor import runner

    monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
    monkeypatch.setenv("HTTPS_PROXY", "http://u:[email protected]")
    monkeypatch.setattr(EnvironmentFactory, "run_preflight", lambda *_args, **_kwargs: None)
    monkeypatch.setattr(
        runner,
        "_check_ack_cluster_readiness_subprocess",
        lambda _kwargs, **_options: (_ for _ in ()).throw(RuntimeError("API rejected user u password pw")),
    )

    rendered = " ".join(
        _check_prerequisites(
            env_mode="ack",
            agents=[],
            environment_kwargs={"namespace": "skill-evals"},
        )
    )

    assert " user u " not in rendered
    assert " password pw" not in rendered
    assert "<redacted>" in rendered


def test_ack_subprocess_probe_uses_exact_harbor_environment_and_stdin(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import runner

    captured: dict[str, object] = {}
    child_env = {
        "PATH": "/usr/bin",
        "KUBECONFIG": "/config/ack",
        "KUBERNETES_SERVICE_HOST": "kubernetes.default.svc",
        "KUBERNETES_SERVICE_PORT": "443",
    }
    monkeypatch.setenv("ALIBABA_CLOUD_ACCESS_KEY_ID", "parent-only-credential")

    class FakeProcess:
        pid = 1234
        returncode = 0

        def communicate(self, input_text: str, timeout: int) -> tuple[str, str]:
            captured["input"] = input_text
            captured["timeout"] = timeout
            return "", ""

    def fake_popen(command: list[str], **kwargs: object) -> FakeProcess:
        captured["command"] = list(command)
        captured.update(kwargs)
        return FakeProcess()

    monkeypatch.setattr(runner.subprocess, "Popen", fake_popen)

    runner._check_ack_cluster_readiness_subprocess(
        {
            "namespace": "skill-evals",
            "context": "production",
            "kubeconfig": "/config/ack",
            "node_selector": {"pool": "sandbox"},
        },
        subprocess_env=child_env,
    )

    assert captured["env"] == {**child_env, "HARBOR_TELEMETRY": "0", "PYTHON_DOTENV_DISABLED": "1"}
    assert "ALIBABA_CLOUD_ACCESS_KEY_ID" not in captured["env"]
    assert Path(str(captured["cwd"])) != Path.cwd()
    assert json.loads(str(captured["input"])) == {
        "namespace": "skill-evals",
        "context": "production",
        "kubeconfig": "/config/ack",
    }
    assert captured["timeout"] == runner._ACK_CLUSTER_READINESS_SUBPROCESS_TIMEOUT_SECONDS
    assert captured["text"] is True
    assert captured["stdin"] is subprocess.PIPE
    assert captured["stdout"] is subprocess.PIPE
    assert captured["stderr"] is subprocess.PIPE
    assert captured["start_new_session"] is (os.name == "posix")
    command = captured["command"]
    assert isinstance(command, list)
    assert command[:3] == [sys.executable, "-P", "-c"]
    assert "production" not in " ".join(command)
    assert "/config/ack" not in " ".join(command)


def test_ack_subprocess_probe_redacts_config_values_and_child_credentials(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import runner

    context = "sensitive-context"
    kubeconfig = "/private/ack-config"
    child_env = {"PATH": "/usr/bin", "KUBECONFIG": kubeconfig, "HTTPS_PROXY": "http://u:[email protected]"}

    class FailedProcess:
        pid = 1234
        returncode = 1

        def communicate(self, _input: str, timeout: int) -> tuple[str, str]:
            assert timeout == runner._ACK_CLUSTER_READINESS_SUBPROCESS_TIMEOUT_SECONDS
            return (
                "",
                f"namespaced pod list forbidden for context {context} via {kubeconfig}; proxy user u password pw",
            )

    monkeypatch.setattr(runner.subprocess, "Popen", lambda *_args, **_kwargs: FailedProcess())

    with pytest.raises(RuntimeError) as exc_info:
        runner._check_ack_cluster_readiness_subprocess(
            {"namespace": "skill-evals", "context": context, "kubeconfig": kubeconfig},
            subprocess_env=child_env,
        )

    rendered = str(exc_info.value)
    assert context not in rendered
    assert kubeconfig not in rendered
    assert " user u " not in rendered
    assert " password pw" not in rendered
    assert "namespaced pod list forbidden" in rendered
    assert "<redacted>" in rendered


@pytest.mark.skipif(os.name != "posix", reason="requires POSIX process groups")
def test_ack_subprocess_probe_timeout_kills_the_exec_auth_process_group(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import runner

    signals: list[tuple[int, signal.Signals]] = []

    class HungProcess:
        pid = 4321
        returncode: int | None = None

        def communicate(self, _input: str, timeout: int) -> tuple[str, str]:
            raise subprocess.TimeoutExpired([sys.executable, "-c", "probe"], timeout)

        def wait(self, timeout: int) -> int:
            assert timeout == runner._ACK_CLUSTER_READINESS_REAP_TIMEOUT_SECONDS
            self.returncode = -int(signal.SIGKILL)
            return self.returncode

        def kill(self) -> None:
            raise AssertionError("POSIX cleanup must kill the whole process group")

    monkeypatch.setattr(runner.subprocess, "Popen", lambda *_args, **_kwargs: HungProcess())
    monkeypatch.setattr(runner.os, "killpg", lambda pid, value: signals.append((pid, value)))

    with pytest.raises(RuntimeError, match="timed out after 20 seconds"):
        runner._check_ack_cluster_readiness_subprocess(
            {"namespace": "skill-evals"},
            subprocess_env={"PATH": "/usr/bin", "KUBECONFIG": "/config/ack"},
        )

    assert signals == [(4321, signal.SIGKILL)]


def test_docker_prerequisite_redacts_proxy_uri_and_short_detached_credentials(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skillevaluator.tier3.harbor import runner

    monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
    monkeypatch.setenv("HTTPS_PROXY", "http://u:[email protected]")
    monkeypatch.setattr(
        runner.subprocess,
        "run",
        lambda *_args, **_kwargs: subprocess.CompletedProcess(
            [],
            1,
            stdout="",
            stderr="proxy http://u:[email protected] rejected user u password pw",
        ),
    )

    rendered = " ".join(_check_prerequisites(env_mode="docker", agents=[]))

    assert "http://u:pw@" not in rendered
    assert " user u " not in rendered
    assert " password pw" not in rendered
    assert "<redacted>" in rendered


def test_harbor_preflight_exception_redacts_detached_proxy_credentials(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from harbor.environments.factory import EnvironmentFactory

    from skillevaluator.tier3.harbor import runner

    monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
    monkeypatch.setenv("HTTPS_PROXY", "http://u:[email protected]")
    monkeypatch.setattr(
        EnvironmentFactory,
        "run_preflight",
        lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("proxy rejected user u password pw")),
    )

    rendered = " ".join(_check_prerequisites(env_mode="e2b", agents=[]))

    assert " user u " not in rendered
    assert " password pw" not in rendered
    assert "<redacted>" in rendered


def test_harbor_preflight_does_not_swallow_keyboard_interrupt(monkeypatch: pytest.MonkeyPatch) -> None:
    from harbor.environments.factory import EnvironmentFactory

    monkeypatch.setattr(
        "skillevaluator.tier3.harbor.runner.subprocess.run",
        lambda *_args, **_kwargs: subprocess.CompletedProcess([], 0, stdout="Docker Compose version v2", stderr=""),
    )
    monkeypatch.setattr(
        EnvironmentFactory,
        "run_preflight",
        lambda *_args, **_kwargs: (_ for _ in ()).throw(KeyboardInterrupt()),
    )

    with pytest.raises(KeyboardInterrupt):
        _check_prerequisites(env_mode="docker", agents=[])


def test_docker_prerequisite_rejects_missing_compose_v2(monkeypatch: pytest.MonkeyPatch) -> None:
    from harbor.environments.factory import EnvironmentFactory

    monkeypatch.setattr(EnvironmentFactory, "run_preflight", lambda *_args, **_kwargs: None)
    monkeypatch.setattr(
        "skillevaluator.tier3.harbor.runner.subprocess.run",
        lambda *_args, **_kwargs: subprocess.CompletedProcess([], 1, stdout="", stderr="unknown command: compose"),
    )

    errors = _check_prerequisites(env_mode="docker", agents=[])

    assert len(errors) == 1
    assert "Docker Compose v2" in errors[0]


def test_docker_prerequisite_accepts_compose_v2(monkeypatch: pytest.MonkeyPatch) -> None:
    from harbor.environments.factory import EnvironmentFactory

    monkeypatch.setattr(EnvironmentFactory, "run_preflight", lambda *_args, **_kwargs: None)
    monkeypatch.setattr(
        "skillevaluator.tier3.harbor.runner.subprocess.run",
        lambda *_args, **_kwargs: subprocess.CompletedProcess(
            [], 0, stdout="Docker Compose version v2.40.0", stderr=""
        ),
    )

    assert _check_prerequisites(env_mode="docker", agents=[]) == []


def test_local_nvidia_opencode_config_disables_oauth_for_remote_mcp(monkeypatch: pytest.MonkeyPatch) -> None:
    from types import SimpleNamespace

    agent = object.__new__(SkillEvaluatorLocalOpenCode)
    agent.mcp_servers = [
        SimpleNamespace(name="remote-tools", transport="streamable-http", url="https://tools.example/mcp"),
        SimpleNamespace(name="local-tools", transport="stdio", command="tool-server", args=["--stdio"]),
    ]
    agent._opencode_config = {}
    monkeypatch.setenv("OPENAI_BASE_URL", "https://integrate.api.nvidia.com/v1")

    command = agent._nvidia_provider_config_command("nvidia/model")

    payload = shlex.split(command.split("&&", 1)[1])[1]
    config = json.loads(payload)
    assert config["mcp"]["remote-tools"] == {"type": "remote", "url": "https://tools.example/mcp", "oauth": False}
    assert config["mcp"]["local-tools"] == {"type": "local", "command": ["tool-server", "--stdio"]}


@pytest.mark.parametrize("script_name", ["harbor", "harbor.exe", "harbor.cmd"])
def test_harbor_bin_resolves_windows_console_script_suffixes(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
    script_name: str,
) -> None:
    """The Harbor console script carries a suffix on Windows.

    Probing only the suffix-less ``harbor`` name misses the interpreter-sibling
    install entirely and falls through to ``shutil.which``, so
    ``doctor --env-mode docker`` reports "harbor CLI not found" on a machine
    where Harbor 0.24.0 is installed and runnable.
    """
    from skillevaluator.tier3.harbor import runner

    fake_interpreter = tmp_path / "python.exe"
    fake_interpreter.write_text("", encoding="utf-8")
    (tmp_path / script_name).write_text("", encoding="utf-8")
    monkeypatch.setattr(runner.os, "sys", SimpleNamespace(executable=str(fake_interpreter)))
    monkeypatch.delenv("PATH", raising=False)

    assert runner._harbor_bin() == str(tmp_path / script_name)


def test_harbor_bin_falls_back_to_path_lookup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    """With no sibling script, PATH lookup still applies."""
    from skillevaluator.tier3.harbor import runner

    fake_interpreter = tmp_path / "python.exe"
    fake_interpreter.write_text("", encoding="utf-8")
    monkeypatch.setattr(runner.os, "sys", SimpleNamespace(executable=str(fake_interpreter)))
    monkeypatch.setattr(runner.shutil, "which", lambda _name: "/opt/harbor/bin/harbor")

    assert runner._harbor_bin() == "/opt/harbor/bin/harbor"


def test_harbor_bin_returns_bare_name_when_missing_everywhere(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    """A missing Harbor still yields the command name for the caller to report."""
    from skillevaluator.tier3.harbor import runner

    fake_interpreter = tmp_path / "python.exe"
    fake_interpreter.write_text("", encoding="utf-8")
    monkeypatch.setattr(runner.os, "sys", SimpleNamespace(executable=str(fake_interpreter)))
    monkeypatch.setattr(runner.shutil, "which", lambda _name: None)

    assert runner._harbor_bin() == "harbor"