Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
15.5 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

import importlib.util
import json
from pathlib import Path
from types import ModuleType

import pytest
import yaml

REPO = Path(__file__).resolve().parents[1]
GENERATOR = REPO / "benchmark" / "prepare_harbor_dataset.py"


def _load_generator_module() -> ModuleType:
    spec = importlib.util.spec_from_file_location("switchyard_prepare_harbor_dataset", GENERATOR)
    assert spec is not None
    assert spec.loader is not None
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def _write_task(root: Path, name: str, task_toml: str, dockerfile: str | None = None) -> Path:
    task = root / name
    env = task / "environment"
    env.mkdir(parents=True)
    (task / "task.toml").write_text(task_toml)
    if dockerfile is not None:
        (env / "Dockerfile").write_text(dockerfile)
    return task


def _prepare(
    tmp_path: Path,
    source: Path,
    source_dataset: str = "[email protected]",
) -> Path:
    module = _load_generator_module()
    output = tmp_path / "prepared"
    return module.prepare_dataset(
        source_dataset=source_dataset,
        source_dir=source,
        output_dir=output,
        harbor_command="harbor",
        overwrite=False,
    )


def test_find_exported_dataset_root_uses_harbor_package_short_name(tmp_path: Path) -> None:
    module = _load_generator_module()
    download_root = tmp_path / "_downloads"
    _write_task(download_root / "terminal-bench-2", "example-task", "[environment]\n")

    found = module._find_exported_dataset_root(
        download_root,
        "terminal-bench/terminal-bench-2",
    )

    assert found == download_root / "terminal-bench-2"


def test_find_exported_dataset_root_keeps_legacy_dataset_name(tmp_path: Path) -> None:
    module = _load_generator_module()
    download_root = tmp_path / "_downloads"
    _write_task(download_root / "openthoughts-tblite", "example-task", "[environment]\n")

    found = module._find_exported_dataset_root(download_root, "[email protected]")

    assert found == download_root / "openthoughts-tblite"


def test_find_exported_dataset_root_ignores_other_exported_datasets(tmp_path: Path) -> None:
    module = _load_generator_module()
    download_root = tmp_path / "_downloads"
    _write_task(download_root / "openthoughts-tblite", "lite-task", "[environment]\n")
    _write_task(download_root / "terminal-bench-2", "tb2-task", "[environment]\n")

    found = module._find_exported_dataset_root(
        download_root,
        "terminal-bench/terminal-bench-2",
    )

    assert found == download_root / "terminal-bench-2"


def test_terminal_bench_2_dataset_adds_proxy_allowlist_hosts(tmp_path: Path) -> None:
    source = tmp_path / "source"
    _write_task(source, "tb2-task", "[environment]\n", "FROM ubuntu:22.04\n")

    output = _prepare(
        tmp_path,
        source,
        source_dataset="terminal-bench/terminal-bench-2",
    )
    allowlist = (output / "tb2-task" / "environment" / "proxy" / "allowlist-base.txt").read_text()
    manifest = json.loads((output / "switchyard_dataset_manifest.json").read_text())

    for host in (
        "archive.ubuntu.com",
        "deb.debian.org",
        "pypi.org",
        "files.pythonhosted.org",
        "github.com",
        "huggingface.co",
        "download.pytorch.org",
        "download-r2.pytorch.org",
        "cloud.r-project.org",
        "www.cs.toronto.edu",
        "www.rcsb.org",
    ):
        assert host in allowlist
        assert host in manifest["closed_book"]["proxy_allowlist_hosts"]


def test_terminal_bench_2_1_dataset_reuses_terminal_bench_2_allowlist(tmp_path: Path) -> None:
    source = tmp_path / "source"
    _write_task(source, "tb21-task", "[environment]\n", "FROM ubuntu:22.04\n")

    output = _prepare(
        tmp_path,
        source,
        source_dataset="terminal-bench/terminal-bench-2-1",
    )
    allowlist = (output / "tb21-task" / "environment" / "proxy" / "allowlist-base.txt").read_text()
    manifest = json.loads((output / "switchyard_dataset_manifest.json").read_text())

    for host in (
        "archive.ubuntu.com",
        "pypi.org",
        "github.com",
        "huggingface.co",
        "download.pytorch.org",
    ):
        assert host in allowlist
        assert host in manifest["closed_book"]["proxy_allowlist_hosts"]


def test_swe_bench_pro_dataset_keeps_agent_proxy_allowlist_empty(tmp_path: Path) -> None:
    source = tmp_path / "source"
    _write_task(source, "swe-task", "[environment]\n", "FROM ubuntu:22.04\n")

    output = _prepare(
        tmp_path,
        source,
        source_dataset="cais/swebenchpro",
    )
    allowlist = (output / "swe-task" / "environment" / "proxy" / "allowlist-base.txt").read_text()
    manifest = json.loads((output / "switchyard_dataset_manifest.json").read_text())

    for host in (
        "pypi.org",
        "github.com",
        "registry.npmjs.org",
    ):
        assert host not in allowlist
    assert manifest["closed_book"]["proxy_allowlist_hosts"] == []


def test_legacy_dataset_keeps_proxy_allowlist_empty(tmp_path: Path) -> None:
    source = tmp_path / "source"
    _write_task(source, "lite-task", "[environment]\n", "FROM ubuntu:22.04\n")

    output = _prepare(tmp_path, source)
    allowlist = (output / "lite-task" / "environment" / "proxy" / "allowlist-base.txt").read_text()
    manifest = json.loads((output / "switchyard_dataset_manifest.json").read_text())

    assert "pypi.org" not in allowlist
    assert "archive.ubuntu.com" not in allowlist
    assert manifest["closed_book"]["proxy_allowlist_hosts"] == []


def test_prebuilt_docker_image_task_becomes_derived_dockerfile(tmp_path: Path) -> None:
    source = tmp_path / "source"
    _write_task(
        source,
        "prebuilt-task",
        '[environment]\ndocker_image = "python:3.12-slim"\n',
    )

    output = _prepare(tmp_path, source)
    task = output / "prebuilt-task"

    assert "docker_image" not in (task / "task.toml").read_text()
    dockerfile = (task / "environment" / "Dockerfile").read_text()
    assert dockerfile.startswith("FROM python:3.12-slim\nUSER root\n")
    assert "@anthropic-ai/[email protected]" in dockerfile
    assert "@openai/[email protected]" in dockerfile
    assert "[email protected]" in dockerfile
    assert "@earendil-works/[email protected]" in dockerfile


def test_dockerfile_only_task_gets_prebake_layer(tmp_path: Path) -> None:
    source = tmp_path / "source"
    _write_task(
        source,
        "dockerfile-task",
        "[environment]\n",
        "FROM ubuntu:22.04\nRUN echo task\n",
    )

    output = _prepare(tmp_path, source)
    dockerfile = (output / "dockerfile-task" / "environment" / "Dockerfile").read_text()

    assert dockerfile.startswith("FROM ubuntu:22.04\nRUN echo task\n")
    assert "SWITCHYARD_PREBAKED_AGENT_VERSIONS" in dockerfile
    assert "/usr/local/lib/node_modules/npm" in dockerfile
    assert "node-v20.11.1-linux-$node_arch.tar.gz" in dockerfile


def test_generated_compose_contains_closed_book_proxy_topology(tmp_path: Path) -> None:
    source = tmp_path / "source"
    _write_task(
        source,
        "compose-task",
        "[environment]\n",
        "FROM ubuntu:22.04\n",
    )

    output = _prepare(tmp_path, source)
    dockerfile = (output / "compose-task" / "environment" / "Dockerfile").read_text()
    entrypoint = output / "compose-task" / "environment" / "switchyard-agent-entrypoint.sh"

    assert entrypoint.is_file()
    entrypoint_text = entrypoint.read_text()
    assert 'PROXY_CA="/etc/proxy-ca/ca-cert.pem"' in entrypoint_text
    assert "test -f" in entrypoint_text
    assert "update-ca-certificates" in entrypoint_text
    assert "SWITCHYARD_PROXY_CA" not in entrypoint_text
    assert "update-ca-trust" not in entrypoint_text
    assert "skipping CA install" not in entrypoint_text
    assert "COPY switchyard-agent-entrypoint.sh" in dockerfile
    assert 'ENTRYPOINT ["/usr/local/bin/switchyard-agent-entrypoint.sh"]' in dockerfile

    compose = yaml.safe_load(
        (output / "compose-task" / "environment" / "docker-compose.yaml").read_text()
    )

    assert {"main", "proxy"} <= set(compose["services"])
    assert compose["services"]["main"]["networks"] == ["agent-internal"]
    assert "switchyard-egress" in compose["services"]["proxy"]["networks"]
    assert "agent-internal" in compose["services"]["proxy"]["networks"]
    assert "extra_hosts" not in compose["services"]["proxy"]
    assert compose["networks"]["agent-internal"]["internal"] is True
    assert compose["networks"]["switchyard-egress"] == {
        "external": True,
        "name": "${SWITCHYARD_DOCKER_NETWORK:?set SWITCHYARD_DOCKER_NETWORK}",
    }
    assert "proxy-ca-public" in compose["volumes"]
    main_env = "\n".join(compose["services"]["main"]["environment"])
    assert "NODE_EXTRA_CA_CERTS=/etc/proxy-ca/ca-cert.pem" in main_env
    assert "REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt" in main_env
    assert "SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt" in main_env
    assert "CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt" in main_env
    assert "GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt" in main_env
    proxy_env = "\n".join(compose["services"]["proxy"]["environment"])
    assert "OPENAI_BASE_URL=" in proxy_env
    assert "ANTHROPIC_BASE_URL=" in proxy_env
    assert "ALLOWED_HOSTS=" in proxy_env
    assert "VERIFIER_PROXY_TOKEN=${SWITCHYARD_VERIFIER_PROXY_TOKEN:-}" in proxy_env
    assert "SWITCHYARD_HOST_SOCKET" not in proxy_env
    proxy_assets = output / "compose-task" / "environment" / "proxy"
    assert (proxy_assets / "Dockerfile").is_file()
    assert (proxy_assets / "entrypoint.sh").is_file()
    assert (proxy_assets / "rewriter.py").is_file()
    assert not (proxy_assets / "verifier_proxy.py").exists()
    healthcheck = "\n".join(compose["services"]["proxy"]["healthcheck"]["test"])
    assert "3128" in healthcheck
    assert "3129" in healthcheck


def test_generated_dataset_manifest_records_pins_tasks_and_digests(tmp_path: Path) -> None:
    source = tmp_path / "source"
    _write_task(source, "task-a", "[environment]\n", "FROM ubuntu:22.04\n")
    _write_task(source, "task-b", "[environment]\n", "FROM ubuntu:22.04\n")

    output = _prepare(tmp_path, source)
    manifest = json.loads((output / "switchyard_dataset_manifest.json").read_text())

    assert manifest["source_dataset"] == "[email protected]"
    assert manifest["task_count"] == 2
    assert manifest["agent_versions"] == {
        "CLAUDE_CODE_VERSION": "2.1.211",
        "CODEX_VERSION": "0.144.5",
        "HERMES_VERSION": "3c27eb6234bf91b8ceee9e9071591b31e9b148cb",
        "NODE_VERSION": "20.11.1",
        "OPENCODE_VERSION": "1.18.3",
        "PI_VERSION": "0.84.3",
    }
    assert manifest["closed_book"]["proxy_asset_digest"].startswith("sha256:")
    assert manifest["closed_book"]["verifier_egress"] == "open-via-authenticated-proxy"
    assert {task["name"] for task in manifest["tasks"]} == {"task-a", "task-b"}
    assert all(task["dockerfile_digest"].startswith("sha256:") for task in manifest["tasks"])
    assert all(task["compose_digest"].startswith("sha256:") for task in manifest["tasks"])


def test_generated_compose_bakes_task_id_into_proxy_env(tmp_path: Path) -> None:
    source = tmp_path / "source"
    _write_task(source, "task-id-check", "[environment]\n", "FROM ubuntu:22.04\n")

    output = _prepare(tmp_path, source)
    compose = yaml.safe_load(
        (output / "task-id-check" / "environment" / "docker-compose.yaml").read_text()
    )

    proxy_env = "\n".join(compose["services"]["proxy"]["environment"])
    assert "SWITCHYARD_TASK_ID=task-id-check" in proxy_env
    assert "SWITCHYARD_TRIAL_DIR=${HOST_AGENT_LOGS_PATH:-}" in proxy_env

def test_a_hermes_ref_that_is_not_a_commit_sha_is_rejected() -> None:
    """Only a full commit SHA can be recorded as a pin.

    The dataset manifest presents HERMES_VERSION as a reproducibility guarantee. Two
    builds recording the same string while installing different Hermes code is worse
    than recording nothing, so the build fails rather than asserting a pin it does not
    have. Tags are rejected alongside branches: a tag can be deleted or repointed, so
    it reads as immutable without being so.
    """
    base = {
        "CLAUDE_CODE_VERSION": "1",
        "CODEX_VERSION": "2",
        "OPENCODE_VERSION": "3",
        "PI_VERSION": "5",
        "NODE_VERSION": "4",
    }
    rejected = (
        "main",
        "master",
        "HEAD",
        "v2026.8.3",
        "release/2026.8",
        "3c27eb6",
        "3C27EB6234BF91B8CEEE9E9071591B31E9B148CB",
        "3c27eb6234bf91b8ceee9e9071591b31e9b148cbb",
    )
    for ref in rejected:
        with pytest.raises(SystemExit, match="commit SHA"):
            _load_generator_module()._install_layer({**base, "HERMES_VERSION": ref})


def test_the_hermes_installer_is_fetched_at_the_pinned_commit() -> None:
    """Pinning the agent but running main's installer reintroduces the same drift."""
    sha = "3c27eb6234bf91b8ceee9e9071591b31e9b148cb"
    pins = {
        "CLAUDE_CODE_VERSION": "1",
        "CODEX_VERSION": "2",
        "OPENCODE_VERSION": "3",
        "PI_VERSION": "5",
        "NODE_VERSION": "4",
        "HERMES_VERSION": sha,
    }

    layer = _load_generator_module()._install_layer(pins)

    assert f"hermes-agent/{sha}/scripts/install.sh" in layer
    assert "hermes-agent/main/" not in layer


def test_the_hermes_pin_is_applied_by_commit_and_forced() -> None:
    """`--branch` reaches `git clone --branch`, which rejects a SHA outright.

    `--force-commit` is what makes the pin take effect. Without it the installer skips
    the checkout whenever the commit is an ancestor of the freshly cloned HEAD, warns,
    and leaves the image on the tip of main — the drift the pin exists to prevent,
    arriving as a warning rather than a build failure.
    """
    sha = "3c27eb6234bf91b8ceee9e9071591b31e9b148cb"
    pins = {
        "CLAUDE_CODE_VERSION": "1",
        "CODEX_VERSION": "2",
        "OPENCODE_VERSION": "3",
        "PI_VERSION": "5",
        "NODE_VERSION": "4",
        "HERMES_VERSION": sha,
    }

    layer = _load_generator_module()._install_layer(pins)

    assert f"--commit {sha}" in layer
    assert "--force-commit" in layer
    assert "--branch" not in layer


def test_the_alpine_branch_installs_the_shell_the_installer_needs() -> None:
    """The installer is piped into bash, which Alpine does not ship by default."""
    pins = {
        "CLAUDE_CODE_VERSION": "1",
        "CODEX_VERSION": "2",
        "OPENCODE_VERSION": "3",
        "PI_VERSION": "5",
        "NODE_VERSION": "4",
        "HERMES_VERSION": "3c27eb6234bf91b8ceee9e9071591b31e9b148cb",
    }

    layer = _load_generator_module()._install_layer(pins)

    assert "apk add --no-cache bash git ripgrep xz" in layer


def test_a_missing_hermes_pin_is_reported_with_the_other_pins(tmp_path: Path) -> None:
    """Absent, it must fail the shared pin check rather than crash reading the layer."""
    module = _load_generator_module()
    versions = tmp_path / "agent-versions.env"
    versions.write_text(
        "CLAUDE_CODE_VERSION=1\nCODEX_VERSION=2\nOPENCODE_VERSION=3\nPI_VERSION=5\nNODE_VERSION=4\n"
    )
    module.AGENT_VERSIONS_FILE = versions
    source = tmp_path / "source"
    _write_task(source, "task-a", "[environment]\n", "FROM ubuntu:22.04\n")

    with pytest.raises(ValueError, match="missing pins.*HERMES_VERSION"):
        module.prepare_dataset(
            source_dataset="[email protected]",
            source_dir=source,
            output_dir=tmp_path / "prepared",
            harbor_command="harbor",
            overwrite=False,
        )