Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
8.7 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

from __future__ import annotations

import importlib.util
import json
from copy import deepcopy
from pathlib import Path
from types import ModuleType
from typing import Any

import pytest

BIG = "nvidia_nim/nvidia/nemotron-3-ultra-550b-a55b"
SMALL = "nvidia_nim/nvidia/nemotron-3.5-lightning-30b-a3b"


@pytest.fixture
def comparator() -> ModuleType:
    """Load the offline comparator without the model-serving dependencies."""
    path = Path(__file__).resolve().parents[1] / "benchmark/nemo_gym/compare.py"
    spec = importlib.util.spec_from_file_location("switchyard_nemo_gym_compare", path)
    assert spec is not None and spec.loader is not None
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


@pytest.fixture
def artifacts() -> dict[str, dict[str, Any]]:
    """Build paired runs with one correct and one wrong answer per condition."""
    runs = {}
    for route in ("fixed", "routed"):
        inputs, rows, models = [], [], []
        for index in range(2):
            task = {
                "_ng_task_index": index,
                "_ng_rollout_index": 0,
                "expected_answer": "B",
                "responses_create_params": {
                    "input": [{"role": "user", "content": f"Question {index}"}],
                    "temperature": 0,
                    "max_output_tokens": 8192,
                },
            }
            response_id = f"{route}-{index}"
            inputs.append(task)
            rows.append(
                {
                    **deepcopy(task),
                    "reward": 1 - index,
                    "response": {
                        "id": response_id,
                        "status": "completed",
                        "output": [
                            {
                                "type": "message",
                                "role": "assistant",
                                "content": [
                                    {
                                        "type": "output_text",
                                        "text": "\\boxed{B}" if index == 0 else "\\boxed{A}",
                                    }
                                ],
                            }
                        ],
                    },
                    "ng_perf": {"total_latency_ms": 100 + 200 * index},
                    "ng_model_call_capture": {
                        "calls": [
                            {
                                "model_call_id": response_id,
                                "response_id": response_id,
                                "model": route,
                                "status_code": 200,
                                "error_category": None,
                                "response_status": "completed",
                                "tokens_in": 10 + 10 * index,
                                "tokens_out": 20,
                            }
                        ],
                    },
                }
            )
            models.append(
                {
                    "response_id": response_id,
                    "model": SMALL if route == "routed" and index == 0 else BIG,
                }
            )
        runs[route] = {
            "rollouts_materialized_inputs.jsonl": inputs,
            "rollouts.jsonl": rows,
            "rollouts_failures.jsonl": [],
            "models.jsonl": models,
        }
    return runs


def _write_runs(tmp_path: Path, artifacts: dict[str, dict[str, Any]]) -> list[str]:
    for route, files in artifacts.items():
        directory = tmp_path / route
        directory.mkdir()
        for name, rows in files.items():
            (directory / name).write_text(
                "".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8"
            )
    return [str(tmp_path)]


def _assert_metric(output: str, metric: str, values: list[str]) -> None:
    line = next(line for line in output.splitlines() if line[:29].rstrip() == metric)
    assert line[29:].split() == values


def test_complete_reordered_pair(
    tmp_path: Path,
    comparator: ModuleType,
    artifacts: dict[str, dict[str, Any]],
    capsys: pytest.CaptureFixture[str],
) -> None:
    """Pair by identity, not file order, and report Gym usage and serving models."""
    for rows in artifacts["routed"].values():
        rows.reverse()
    assert comparator.main(_write_runs(tmp_path, artifacts)) == 0
    output = capsys.readouterr()
    assert output.err == ""
    assert "Pairing: matched=2, fixed-only=0, routed-only=0" in output.out
    assert f'fixed serving models: {{"{BIG}": 2}}' in output.out
    assert f"routed serving models: {json.dumps({SMALL: 1, BIG: 1}, sort_keys=True)}" in output.out
    for metric, values in {
        "Paired rollouts": ["2", "2"],
        "Incomplete responses": ["0", "0"],
        "Mean reward": ["0.500", "0.500"],
        "Captured input tokens": ["30", "30"],
        "Captured output tokens": ["40", "40"],
        "Mean rollout latency (ms)": ["200", "200"],
        "Captured calls": ["2", "2"],
        "Captured failed calls": ["0", "0"],
        "Calls with unknown usage": ["0", "0"],
    }.items():
        _assert_metric(output.out, metric, values)
    assert "WARNING" not in output.out
    assert "Gym-captured" in output.out
    assert "Gateway-reported" not in output.out


@pytest.mark.parametrize(
    ("problem", "expected_error"),
    [
        ("incomplete", "Incomplete runs"),
        ("mismatched", "Task inputs, verifier metadata, or generation settings differ"),
        ("capture", "incomplete model-call capture"),
    ],
)
def test_invalid_evidence_never_prints_averages(
    tmp_path: Path,
    comparator: ModuleType,
    artifacts: dict[str, dict[str, Any]],
    capsys: pytest.CaptureFixture[str],
    problem: str,
    expected_error: str,
) -> None:
    """Reject misleading comparisons with a useful diagnostic, not partial averages."""
    routed = artifacts["routed"]
    if problem == "incomplete":
        for run in artifacts.values():
            run["rollouts.jsonl"].pop()
    elif problem == "mismatched":
        routed["rollouts_materialized_inputs.jsonl"][0]["expected_answer"] = "A"
    else:
        routed["rollouts.jsonl"][0]["ng_model_call_capture"]["gaps"] = ["missing exchange"]
    assert comparator.main(_write_runs(tmp_path, artifacts)) == 1
    output = capsys.readouterr()
    assert "Cannot compare:" in output.err
    assert expected_error in output.err
    assert "Mean reward" not in output.out


def test_incomplete_model_response_is_reported(
    tmp_path: Path,
    comparator: ModuleType,
    artifacts: dict[str, dict[str, Any]],
    capsys: pytest.CaptureFixture[str],
) -> None:
    """Keep a Gym-scored truncation in the comparison and make it visible."""
    row = artifacts["routed"]["rollouts.jsonl"][0]
    row["response"]["status"] = "incomplete"
    row["reward"] = 0
    row["ng_model_call_capture"]["calls"][0]["response_status"] = "incomplete"

    assert comparator.main(_write_runs(tmp_path, artifacts)) == 0
    output = capsys.readouterr().out
    _assert_metric(output, "Incomplete responses", ["0", "1"])
    assert "WARNING: routed" in output
    assert "WARNING: fixed" not in output


def test_recovery_keeps_extra_work_and_terminal_attribution(
    tmp_path: Path,
    comparator: ModuleType,
    artifacts: dict[str, dict[str, Any]],
    capsys: pytest.CaptureFixture[str],
) -> None:
    """Count extra captured work without treating unreported failed usage as zero."""
    calls = artifacts["routed"]["rollouts.jsonl"][0]["ng_model_call_capture"]["calls"]
    calls.insert(
        0,
        {
            "model_call_id": "failed",
            "status_code": 503,
            "error_category": "upstream",
            "tokens_in": None,
            "tokens_out": None,
        },
    )
    calls.append(
        {
            **calls[1],
            "model_call_id": "extra",
            "response_id": "superseded",
            "tokens_in": 7,
            "tokens_out": 13,
        }
    )
    artifacts["routed"]["models.jsonl"].append({"response_id": "superseded", "model": BIG})
    assert comparator.main(_write_runs(tmp_path, artifacts)) == 0
    output = capsys.readouterr().out
    for metric, values in {
        "Captured input tokens": ["30", "37+unknown"],
        "Captured output tokens": ["40", "53+unknown"],
        "Captured calls": ["2", "4"],
        "Captured failed calls": ["0", "1"],
        "Calls with unknown usage": ["0", "1"],
    }.items():
        _assert_metric(output, metric, values)
    assert f"routed serving models: {json.dumps({SMALL: 1, BIG: 1}, sort_keys=True)}" in output
    assert "WARNING: routed" in output
    assert "WARNING: fixed" not in output