Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
25.7 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

import json
import os
import shlex
import subprocess
from pathlib import Path

REPO = Path(__file__).resolve().parents[1]
SCRIPT = REPO / "benchmark" / "run-baseline.sh"


def _write_fake_harbor(bin_dir: Path) -> Path:
    harbor = bin_dir / "harbor"
    harbor.write_text(
        """
#!/usr/bin/env bash
set -euo pipefail
if [[ "${1:-}" == "--version" ]]; then
    echo "harbor fake 1.0"
    exit 0
fi
if [[ "${1:-}" == "run" ]]; then
    shift
    jobs_dir=""
    job_name=""
    while [[ $# -gt 0 ]]; do
        case "$1" in
            --jobs-dir) jobs_dir="$2"; shift 2 ;;
            --job-name) job_name="$2"; shift 2 ;;
            *) shift ;;
        esac
    done
    mkdir -p "${jobs_dir}/${job_name}"
    {
        printf 'OPENAI_API_KEY=%s\\n' "${OPENAI_API_KEY:-}"
        printf 'OPENAI_BASE_URL=%s\\n' "${OPENAI_BASE_URL:-}"
        printf 'ANTHROPIC_BASE_URL=%s\\n' "${ANTHROPIC_BASE_URL:-}"
        printf 'ANTHROPIC_AUTH_TOKEN=%s\\n' "${ANTHROPIC_AUTH_TOKEN:-}"
        printf 'ANTHROPIC_API_KEY=%s\\n' "${ANTHROPIC_API_KEY:-}"
        printf 'ANTHROPIC_MODEL=%s\\n' "${ANTHROPIC_MODEL:-}"
        printf 'ANTHROPIC_SMALL_FAST_MODEL=%s\\n' "${ANTHROPIC_SMALL_FAST_MODEL:-}"
        printf 'ANTHROPIC_CUSTOM_MODEL_OPTION=%s\\n' "${ANTHROPIC_CUSTOM_MODEL_OPTION:-}"
        printf 'CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=%s\\n' "${CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY:-}"
        printf 'CLOSED_BOOK_MODE=%s\\n' "${CLOSED_BOOK_MODE:-}"
        printf 'ALLOWED_HOSTS=%s\\n' "${ALLOWED_HOSTS:-}"
        printf 'CODEX_DISABLE_WEB_SEARCH=%s\\n' "${CODEX_DISABLE_WEB_SEARCH:-}"
        printf 'CODEX_MODEL_CATALOG_JSON=%s\\n' "${CODEX_MODEL_CATALOG_JSON:-}"
        printf 'OPENCODE_DISABLE_WEBFETCH=%s\\n' "${OPENCODE_DISABLE_WEBFETCH:-}"
    } > "${jobs_dir}/${job_name}/env.txt"
    printf '%s\\n' '{"stats":{"evals":{"fake":{"n_trials":1,"metrics":[{"mean":1.0}]}}}}' > "${jobs_dir}/${job_name}/result.json"
    exit 0
fi
echo "unexpected harbor args: $*" >&2
exit 2
""".lstrip()
    )
    harbor.chmod(0o755)
    return harbor


def _write_fake_docker(bin_dir: Path) -> Path:
    docker = bin_dir / "docker"
    docker.write_text(
        "#!/usr/bin/env bash\n"
        "if [[ -n \"${FAKE_DOCKER_LOG:-}\" ]]; then\n"
        "    printf '%s\\n' \"$*\" >> \"${FAKE_DOCKER_LOG}\"\n"
        "fi\n"
        "exit 0\n"
    )
    docker.chmod(0o755)
    return docker


def _write_fake_curl(bin_dir: Path) -> Path:
    curl = bin_dir / "curl"
    curl.write_text(
        """
#!/usr/bin/env bash
set -euo pipefail
output=""
url=""
while [[ $# -gt 0 ]]; do
    case "$1" in
        -o) output="$2"; shift 2 ;;
        *) url="$1"; shift ;;
    esac
done
case "${url}" in
    */health) body='{"status":"ok"}' ;;
    */metrics) body='switchyard_total_requests 2' ;;
    */v1/stats) body='{"total_requests":2,"total_errors":0,"models":{"strong":{"calls":2}}}' ;;
    *) exit 22 ;;
esac
if [[ -n "${output}" ]]; then
    printf '%s\\n' "${body}" > "${output}"
else
    printf '%s\\n' "${body}"
fi
""".lstrip()
    )
    curl.chmod(0o755)
    return curl


def _write_fake_harbor_patch(tmp_path: Path) -> Path:
    patch_file = tmp_path / "fake-harbor-agent-patches.diff"
    patch_file.write_text(
        """
--- a/harbor/agents/installed/base.py
+++ b/harbor/agents/installed/base.py
@@ -1 +1 @@
-unpatched
+patched
--- /dev/null
+++ b/harbor/switchyard_patch_id.txt
@@ -0,0 +1 @@
+switchyard-test-patch-v1
""".lstrip()
    )
    return patch_file


def _write_fake_harbor_python(tmp_path: Path, patched: bool = True) -> Path:
    tmp_path.mkdir(parents=True, exist_ok=True)
    harbor_site = tmp_path / "fake-site-packages" / "harbor"
    base = harbor_site / "agents" / "installed" / "base.py"
    base.parent.mkdir(parents=True, exist_ok=True)
    base.write_text("patched\n" if patched else "unpatched\n")
    if patched:
        (harbor_site / "switchyard_patch_id.txt").write_text("switchyard-test-patch-v1\n")

    fake_python = tmp_path / "fake-harbor-python"
    fake_python.write_text(
        f"""#!/usr/bin/env bash
set -euo pipefail
cat >/dev/null
printf '%s\\n' {shlex.quote(str(harbor_site))}
"""
    )
    fake_python.chmod(0o755)
    return fake_python


def _run_baseline(
    tmp_path: Path,
    *args: str,
    env: dict[str, str] | None = None,
    include_dataset: bool = True,
    fake_server: bool = False,
):
    fake_bin = tmp_path / "default-bin"
    fake_bin.mkdir(exist_ok=True)
    _write_fake_harbor(fake_bin)
    if fake_server:
        _write_fake_docker(fake_bin)
        _write_fake_curl(fake_bin)
    fake_patch = _write_fake_harbor_patch(tmp_path)
    fake_harbor_python = _write_fake_harbor_python(tmp_path)
    base_path = env.get("PATH", os.environ["PATH"]) if env else os.environ["PATH"]
    merged_env = {
        "PATH": f"{fake_bin}{os.pathsep}{base_path}",
        "HARBOR_BIN": str(fake_bin / "harbor"),
        "HARBOR_PYTHON": str(fake_harbor_python),
        "OPENROUTER_API_KEY": "or-test",  # pragma: allowlist secret
        "SWITCHYARD_HARBOR_PATCH_FILE": str(fake_patch),
    }
    if env:
        merged_env.update({key: value for key, value in env.items() if key != "PATH"})
    dataset_args: list[str] = []
    if include_dataset and "--harbor-path" not in args:
        dataset = tmp_path / "default-dataset"
        if not dataset.exists():
            _write_closed_book_dataset(dataset)
        dataset_args = ["--harbor-path", str(dataset)]
    return subprocess.run(
        [
            "bash",
            str(SCRIPT),
            "--output-dir",
            str(tmp_path / "out"),
            *dataset_args,
            *args,
        ],
        cwd=REPO,
        env=merged_env,
        text=True,
        capture_output=True,
        check=False,
    )


def _line_argv(stdout: str, prefix: str) -> list[str]:
    line = next(line for line in stdout.splitlines() if line.startswith(prefix))
    return shlex.split(line.removeprefix(prefix))


def _option_value(argv: list[str], option: str) -> str:
    return argv[argv.index(option) + 1]


def _option_values(argv: list[str], option: str) -> list[str]:
    return [argv[idx + 1] for idx, value in enumerate(argv[:-1]) if value == option]


def _write_server_config(path: Path) -> Path:
    path.write_text(
        """
schema_version = 1

[llm_clients.test]
format = "openai_chat"
base_url = "https://example.test/v1"

[targets.strong]
id = "strong-model"
llm_client = "test"

[targets.weak]
id = "weak-model"
llm_client = "test"

[routes.random]
id = "tb-lite-random-routing"
type = "random"
targets = ["strong", "weak"]
weights = [1, 1]
seed = 444
""".lstrip()
    )
    return path


def _write_closed_book_dataset(path: Path) -> Path:
    path.mkdir()
    (path / "switchyard_dataset_manifest.json").write_text(
        json.dumps(
            {
                "source_dataset": "[email protected]",
                "task_count": 1,
                "agent_versions": {"codex": "0.144.5"},
            }
        )
    )
    return path


def test_direct_mode_requires_model(tmp_path: Path) -> None:
    result = _run_baseline(tmp_path, "--dry-run")

    assert result.returncode != 0
    assert "--model is required when running direct upstream" in result.stderr


def test_direct_mode_defaults_to_openrouter_upstream_without_switchyard(tmp_path: Path) -> None:
    result = _run_baseline(
        tmp_path,
        "--model",
        "openai/gpt-5.2",
        "--agent",
        "codex",
        "--dry-run",
    )

    assert result.returncode == 0, result.stderr
    harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
    assert _option_value(harbor, "--model") == "openai/gpt-5.2"
    agent_env = _option_values(harbor, "--ae")
    ca_env_prefixes = ("SSL_CERT_FILE=", "REQUESTS_CA_BUNDLE=", "CURL_CA_BUNDLE=", "GIT_SSL_CAINFO=")
    assert not any(
        value.startswith(ca_env_prefixes) for value in agent_env
    )
    assert "server_preset: direct" in result.stdout
    assert "server_mode:   direct" in result.stdout
    assert "upstream_url:  https://openrouter.ai/api/v1" in result.stdout
    assert "api_key_env:   OPENROUTER_API_KEY" in result.stdout
    assert "SERVER_CMD: <direct-upstream>" in result.stdout
    assert "switchyard-server" not in result.stdout
    assert "server_config:" not in result.stdout


def test_direct_mode_uses_generic_upstream_key_when_set(tmp_path: Path) -> None:
    result = _run_baseline(
        tmp_path,
        "--model",
        "provider/model",
        "--agent",
        "codex",
        "--dry-run",
        env={
            "UPSTREAM_API_KEY": "upstream-test",  # pragma: allowlist secret
            "UPSTREAM_BASE_URL": "https://provider.example/v1",
        },
    )

    assert result.returncode == 0, result.stderr
    assert "upstream_url:  https://provider.example/v1" in result.stdout
    assert "api_key_env:   UPSTREAM_API_KEY" in result.stdout


def test_direct_mode_requires_selected_api_key_env(tmp_path: Path) -> None:
    result = _run_baseline(
        tmp_path,
        "--model",
        "openai/gpt-5.2",
        "--dry-run",
        env={"OPENROUTER_API_KEY": ""},
    )

    assert result.returncode != 0
    assert "direct upstream requires $OPENROUTER_API_KEY to be set" in result.stderr


def test_direct_mode_rejects_server_only_options(tmp_path: Path) -> None:
    result = _run_baseline(
        tmp_path,
        "--model",
        "openai/gpt-5.2",
        "--server-extra",
        "--log-level=debug",
        "--dry-run",
    )

    assert result.returncode != 0
    assert "--server-extra requires --server-config" in result.stderr

    result = _run_baseline(
        tmp_path,
        "--route-model",
        "tb-lite-random-routing",
        "--dry-run",
    )

    assert result.returncode != 0
    assert "--route-model requires --server-config" in result.stderr


def test_legacy_routing_profiles_are_rejected(tmp_path: Path) -> None:
    result = _run_baseline(
        tmp_path,
        "--routing-profiles",
        str(tmp_path / "routes.yaml"),
        "--model",
        "switchyard",
        "--dry-run",
    )

    assert result.returncode != 0
    assert "--routing-profiles is no longer supported" in result.stderr
    assert "use --server-config" in result.stderr


def test_dry_run_claude_code_opus_defaults_high_reasoning(tmp_path: Path) -> None:
    result = _run_baseline(
        tmp_path,
        "--server-config",
        str(REPO / "benchmark" / "server-configs" / "tb-lite-single-opus-4-7.toml"),
        "--route-model",
        "tb-lite-single-opus-4-7",
        "--agent",
        "claude-code",
        "--dry-run",
    )

    assert result.returncode == 0, result.stderr
    harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
    assert _option_value(harbor, "--model") == "tb-lite-single-opus-4-7"
    assert "version=2.1.211" in _option_values(harbor, "--ak")
    assert "reasoning_effort=high" in _option_values(harbor, "--ak")
    assert "thinking=adaptive" in _option_values(harbor, "--ak")
    assert "reasoning:     high" in result.stdout
    assert "harbor_server: http://switchyard:4000" in result.stdout


def test_dry_run_codex_gpt_defaults_high_reasoning(tmp_path: Path) -> None:
    result = _run_baseline(
        tmp_path,
        "--server-config",
        str(REPO / "benchmark" / "server-configs" / "tb-lite-single-gpt-5-5.toml"),
        "--route-model",
        "tb-lite-single-gpt-5-5",
        "--agent",
        "codex",
        "--dry-run",
    )

    assert result.returncode == 0, result.stderr
    harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
    assert _option_value(harbor, "--model") == "tb-lite-single-gpt-5-5"
    assert "version=0.144.5" in _option_values(harbor, "--ak")
    assert "reasoning_effort=high" in _option_values(harbor, "--ak")
    catalog_env = next(
        value
        for value in _option_values(harbor, "--ae")
        if value.startswith("CODEX_MODEL_CATALOG_JSON=")
    )
    assert catalog_env.endswith("/codex_model_catalog.json")
    assert "codex_catalog:" in result.stdout


def test_dry_run_explicit_empty_reasoning_omits_kwarg(tmp_path: Path) -> None:
    profile = _write_server_config(tmp_path / "routes.toml")

    result = _run_baseline(
        tmp_path,
        "--server-config",
        str(profile),
        "--route-model",
        "tb-lite-random-routing",
        "--agent",
        "claude-code",
        "--reasoning-effort",
        "",
        "--dry-run",
    )

    assert result.returncode == 0, result.stderr
    harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
    assert not any(value.startswith("reasoning_effort=") for value in _option_values(harbor, "--ak"))
    assert not any(value.startswith("thinking=") for value in _option_values(harbor, "--ak"))
    assert "reasoning:     unset" in result.stdout


def test_dry_run_explicit_reasoning_overrides_default(tmp_path: Path) -> None:
    profile = _write_server_config(tmp_path / "routes.toml")

    result = _run_baseline(
        tmp_path,
        "--server-config",
        str(profile),
        "--route-model",
        "tb-lite-random-routing",
        "--agent",
        "claude-code",
        "--reasoning-effort",
        "medium",
        "--dry-run",
    )

    assert result.returncode == 0, result.stderr
    harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
    assert "reasoning_effort=medium" in _option_values(harbor, "--ak")
    assert "thinking=adaptive" in _option_values(harbor, "--ak")


def test_dry_run_explicit_claude_thinking_overrides_adaptive_default(tmp_path: Path) -> None:
    profile = _write_server_config(tmp_path / "routes.toml")

    result = _run_baseline(
        tmp_path,
        "--server-config",
        str(profile),
        "--route-model",
        "tb-lite-random-routing",
        "--agent",
        "claude-code",
        "--reasoning-effort",
        "high",
        "--harbor-extra",
        "--ak",
        "--harbor-extra",
        "thinking=disabled",
        "--dry-run",
    )

    assert result.returncode == 0, result.stderr
    harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
    thinking = [value for value in _option_values(harbor, "--ak") if value.startswith("thinking=")]
    assert thinking == ["thinking=disabled"]


def test_harbor_path_is_required(tmp_path: Path) -> None:
    profile = _write_server_config(tmp_path / "routes.toml")

    result = _run_baseline(
        tmp_path,
        "--server-config",
        str(profile),
        "--route-model",
        "tb-lite-random-routing",
        "--dry-run",
        include_dataset=False,
    )

    assert result.returncode != 0
    assert "--harbor-path is required" in result.stderr


def test_closed_book_dataset_rejects_unpatched_harbor(tmp_path: Path) -> None:
    dataset = _write_closed_book_dataset(tmp_path / "dataset")
    profile = _write_server_config(tmp_path / "routes.toml")
    fake_python = _write_fake_harbor_python(tmp_path / "unpatched", patched=False)

    result = _run_baseline(
        tmp_path,
        "--harbor-path",
        str(dataset),
        "--server-config",
        str(profile),
        "--route-model",
        "tb-lite-random-routing",
        "--harbor-python",
        str(fake_python),
        "--dry-run",
        include_dataset=False,
    )

    assert result.returncode != 0
    assert "The current Harbor patch is not applied cleanly" in result.stderr
    assert "run-baseline.sh requires patched Harbor" in result.stderr
    assert "Verification used:" in result.stderr
    assert "patch -p1 <" in result.stderr


def test_dry_run_server_config_uses_rust_switchyard_server(tmp_path: Path) -> None:
    profile = _write_server_config(tmp_path / "routes.toml")

    result = _run_baseline(
        tmp_path,
        "--server-config",
        str(profile),
        "--model",
        "tb-lite-random-routing",
        "--dry-run",
    )

    assert result.returncode == 0, result.stderr
    server = _line_argv(result.stdout, "SERVER_CMD: ")
    harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
    assert server[0] == "switchyard-server"
    assert _option_value(server, "--config") == str(profile)
    assert _option_value(server, "--host") == "0.0.0.0"
    assert _option_value(server, "--port") == "4000"
    assert _option_value(harbor, "--model") == "openai/tb-lite-random-routing"
    assert "server_config:" in result.stdout
    assert "route_model:   tb-lite-random-routing" in result.stdout


def test_foreground_server_run_collects_rust_stats_endpoint(tmp_path: Path) -> None:
    profile = _write_server_config(tmp_path / "routes.toml")
    docker_log = tmp_path / "docker.log"

    result = _run_baseline(
        tmp_path,
        "--server-config",
        str(profile),
        "--route-model",
        "tb-lite-random-routing",
        "--foreground",
        fake_server=True,
        env={"FAKE_DOCKER_LOG": str(docker_log)},
    )

    assert result.returncode == 0, result.stderr
    run_dir = next((tmp_path / "out").glob("baseline-serve-serve-*"))
    manifest = json.loads((run_dir / "run_manifest.json").read_text())
    stats = json.loads((run_dir / "routing_stats_final.json").read_text())

    assert stats["total_requests"] == 2
    assert manifest["outcomes"]["routing_stats_json_status"] == "present"
    assert manifest["outcomes"]["server_metrics_prom_status"] == "present"
    assert "--env NVIDIA_API_KEY" in docker_log.read_text()


def test_dry_run_route_model_alias_uses_rust_switchyard_server(tmp_path: Path) -> None:
    profile = _write_server_config(tmp_path / "routes.toml")

    result = _run_baseline(
        tmp_path,
        "--server-config",
        str(profile),
        "--route-model",
        "tb-lite-random-routing",
        "--dry-run",
    )

    assert result.returncode == 0, result.stderr
    server = _line_argv(result.stdout, "SERVER_CMD: ")
    harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
    assert server[0] == "switchyard-server"
    assert _option_value(server, "--config") == str(profile)
    assert _option_value(harbor, "--model") == "openai/tb-lite-random-routing"
    assert "route_model:   tb-lite-random-routing" in result.stdout


def test_dry_run_server_config_honors_explicit_harbor_model(tmp_path: Path) -> None:
    profile = _write_server_config(tmp_path / "routes.toml")

    result = _run_baseline(
        tmp_path,
        "--server-config",
        str(profile),
        "--route-model",
        "tb-lite-random-routing",
        "--harbor-model",
        "openai/custom-route-label",
        "--dry-run",
    )

    assert result.returncode == 0, result.stderr
    harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
    assert _option_value(harbor, "--model") == "openai/custom-route-label"


def test_dry_run_harbor_extra(tmp_path: Path) -> None:
    profile = _write_server_config(tmp_path / "routes.toml")

    result = _run_baseline(
        tmp_path,
        "--server-config",
        str(profile),
        "--route-model",
        "tb-lite-random-routing",
        "--harbor-extra",
        "--include-task-name=hello",
        "--harbor-extra",
        "--no-upload",
        "--dry-run",
    )

    assert result.returncode == 0, result.stderr
    harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
    assert "--dataset" not in harbor
    assert "--include-task-name=hello" in harbor
    assert "--no-upload" in harbor


def test_dry_run_harbor_path_uses_local_dataset_and_closed_book_artifact(
    tmp_path: Path,
) -> None:
    dataset = _write_closed_book_dataset(tmp_path / "dataset")
    profile = _write_server_config(tmp_path / "routes.toml")

    result = _run_baseline(
        tmp_path,
        "--harbor-path",
        str(dataset),
        "--server-config",
        str(profile),
        "--route-model",
        "tb-lite-random-routing",
        "--agent",
        "codex",
        "--dry-run",
        include_dataset=False,
    )

    assert result.returncode == 0, result.stderr
    harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
    assert "--dataset" not in harbor
    assert _option_value(harbor, "--path") == str(dataset)
    assert _option_value(harbor, "--artifact") == "/etc/proxy-ca/strip.jsonl"
    assert "version=0.144.5" in _option_values(harbor, "--ak")
    assert "CODEX_DISABLE_WEB_SEARCH=1" in _option_values(harbor, "--ae")
    catalog_env = next(
        value
        for value in _option_values(harbor, "--ae")
        if value.startswith("CODEX_MODEL_CATALOG_JSON=")
    )
    assert catalog_env.endswith("/codex_model_catalog.json")
    verifier_env = _option_values(harbor, "--ve")
    assert "HTTP_PROXY=${SWITCHYARD_VERIFIER_HTTP_PROXY}" in verifier_env
    assert "HTTPS_PROXY=${SWITCHYARD_VERIFIER_HTTP_PROXY}" in verifier_env
    assert "NO_PROXY=localhost,127.0.0.1,proxy" in verifier_env
    assert "book_mode:     closed" in result.stdout
    assert "harbor_patch:  verified" in result.stdout
    assert "harbor_server: http://switchyard:4000" in result.stdout
    assert "docker_network:" in result.stdout
    assert "verifier_net: authenticated proxy egress" in result.stdout
    assert f"harbor_path:   {dataset}" in result.stdout


def test_dry_run_open_book_uses_proxy_topology_without_tool_disables(tmp_path: Path) -> None:
    dataset = _write_closed_book_dataset(tmp_path / "dataset")
    profile = _write_server_config(tmp_path / "routes.toml")

    result = _run_baseline(
        tmp_path,
        "--harbor-path",
        str(dataset),
        "--server-config",
        str(profile),
        "--route-model",
        "tb-lite-random-routing",
        "--agent",
        "codex",
        "--book-mode",
        "open",
        "--dry-run",
        include_dataset=False,
    )

    assert result.returncode == 0, result.stderr
    harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
    assert _option_value(harbor, "--path") == str(dataset)
    assert _option_value(harbor, "--artifact") == "/etc/proxy-ca/strip.jsonl"
    assert "CODEX_DISABLE_WEB_SEARCH=1" not in _option_values(harbor, "--ae")
    assert not _option_values(harbor, "--ve")
    assert "book_mode:     open" in result.stdout
    assert "harbor_server: http://switchyard:4000" in result.stdout
    assert "proxy_mode:    open" in result.stdout
    assert "verifier_net: authenticated proxy egress" not in result.stdout


def test_dry_run_claude_closed_book_merges_disallowed_tools(tmp_path: Path) -> None:
    dataset = _write_closed_book_dataset(tmp_path / "dataset")
    profile = _write_server_config(tmp_path / "routes.toml")

    result = _run_baseline(
        tmp_path,
        "--harbor-path",
        str(dataset),
        "--server-config",
        str(profile),
        "--route-model",
        "tb-lite-random-routing",
        "--agent",
        "claude-code",
        "--harbor-extra",
        "--ak",
        "--harbor-extra",
        "disallowed_tools=Bash",
        "--dry-run",
        include_dataset=False,
    )

    assert result.returncode == 0, result.stderr
    harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
    assert "version=2.1.211" in _option_values(harbor, "--ak")
    assert "disallowed_tools=Bash,WebFetch,WebSearch" in _option_values(harbor, "--ak")


def test_dry_run_opencode_closed_book_disables_webfetch(tmp_path: Path) -> None:
    dataset = _write_closed_book_dataset(tmp_path / "dataset")
    profile = _write_server_config(tmp_path / "routes.toml")

    result = _run_baseline(
        tmp_path,
        "--harbor-path",
        str(dataset),
        "--server-config",
        str(profile),
        "--route-model",
        "tb-lite-random-routing",
        "--agent",
        "opencode",
        "--dry-run",
        include_dataset=False,
    )

    assert result.returncode == 0, result.stderr
    harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
    assert _option_value(harbor, "--model") == "nvidia/tb-lite-random-routing"
    assert "version=1.18.3" in _option_values(harbor, "--ak")
    assert "OPENCODE_DISABLE_WEBFETCH=1" in _option_values(harbor, "--ae")


def test_dry_run_pi_uses_switchyard_label_pin_and_thinking(tmp_path: Path) -> None:
    profile = _write_server_config(tmp_path / "routes.toml")

    result = _run_baseline(
        tmp_path,
        "--server-config",
        str(profile),
        "--model",
        "tb-lite-random-routing",
        "--agent",
        "pi",
        "--reasoning-effort",
        "high",
        "--dry-run",
    )

    assert result.returncode == 0, result.stderr
    harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
    assert _option_value(harbor, "--model") == "switchyard/tb-lite-random-routing"
    kwargs = _option_values(harbor, "--ak")
    assert "version=0.84.3" in kwargs
    # Harbor's pi agent has a --thinking flag, not reasoning_effort.
    assert "thinking=high" in kwargs
    assert not any(kwarg.startswith("reasoning_effort=") for kwarg in kwargs)


def test_task_list_file_expands_to_include_task_name(tmp_path: Path) -> None:
    task_list = tmp_path / "tasks.txt"
    task_list.write_text("# comment\nalpha\n\nbeta  # inline\n")
    profile = _write_server_config(tmp_path / "routes.toml")

    result = _run_baseline(
        tmp_path,
        "--server-config",
        str(profile),
        "--route-model",
        "tb-lite-random-routing",
        "--task-list-file",
        str(task_list),
        "--dry-run",
    )

    assert result.returncode == 0, result.stderr
    harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
    assert _option_values(harbor, "--include-task-name") == ["alpha", "beta"]


def test_dry_run_uses_harbor_bin_override(tmp_path: Path) -> None:
    override_bin = tmp_path / "override-bin"
    override_bin.mkdir()
    expected_harbor = _write_fake_harbor(override_bin)
    profile = _write_server_config(tmp_path / "routes.toml")

    result = _run_baseline(
        tmp_path,
        "--server-config",
        str(profile),
        "--route-model",
        "tb-lite-random-routing",
        "--dry-run",
        env={"HARBOR_BIN": str(expected_harbor)},
    )

    assert result.returncode == 0, result.stderr
    harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
    assert harbor[0] == str(expected_harbor)
    assert f"harbor_cmd:    {expected_harbor} (override)" in result.stdout