Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
6.6 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

import json
import sys
import time
from pathlib import Path

import pytest

import scripts.aiperf_runner as aiperf_runner
from scripts.aiperf_runner import aggregate_exports, run_profile, validate_aiperf_version

_ALL_FAILED = "All 2 inference request(s) failed; no successful responses were collected."


def _write_stubborn_worker(worker) -> None:
    worker.write_text(
        """import signal
import sys
import time
from pathlib import Path

signal.signal(signal.SIGTERM, signal.SIG_IGN)
heartbeat = Path(sys.argv[1])
while True:
    heartbeat.write_text(str(time.monotonic()))
    time.sleep(0.05)
"""
    )


def _assert_heartbeat_stopped(heartbeat) -> None:
    heartbeat_before = heartbeat.read_text()
    time.sleep(0.2)
    assert heartbeat.read_text() == heartbeat_before


def test_run_profile_timeout_kills_the_worker_process_group(tmp_path, monkeypatch) -> None:
    fake = tmp_path / "fake_aiperf.py"
    worker = tmp_path / "worker.py"
    child_pid = tmp_path / "child-pid"
    heartbeat = tmp_path / "heartbeat"
    _write_stubborn_worker(worker)
    monkeypatch.setattr(aiperf_runner, "PROCESS_GROUP_GRACE_SECONDS", 0.2)
    fake.write_text(
        """import subprocess
import sys
import time
from pathlib import Path

child = subprocess.Popen([sys.executable, sys.argv[2], sys.argv[3]])
Path(sys.argv[1]).write_text(str(child.pid))
time.sleep(60)
"""
    )

    started_at = time.monotonic()
    with pytest.raises(RuntimeError, match="exceeded its 1-second run limit"):
        run_profile(
            [sys.executable, str(fake), str(child_pid), str(worker), str(heartbeat)],
            tmp_path / "aiperf.log",
            tmp_path / "artifacts",
            timeout_seconds=1,
        )

    assert time.monotonic() - started_at < 10
    assert child_pid.read_text().isdigit()
    _assert_heartbeat_stopped(heartbeat)
    assert "exceeded its 1-second run limit" in (tmp_path / "aiperf.log").read_text()


def test_run_profile_stops_workers_after_the_leader_exits(tmp_path, monkeypatch) -> None:
    fake = tmp_path / "fake_aiperf.py"
    worker = tmp_path / "worker.py"
    child_pid = tmp_path / "child-pid"
    heartbeat = tmp_path / "heartbeat"
    _write_stubborn_worker(worker)
    monkeypatch.setattr(aiperf_runner, "PROCESS_GROUP_GRACE_SECONDS", 0.2)
    fake.write_text(
        """import subprocess
import sys
import time
from pathlib import Path

child = subprocess.Popen([sys.executable, sys.argv[2], sys.argv[3]])
Path(sys.argv[1]).write_text(str(child.pid))
while not Path(sys.argv[3]).exists():
    time.sleep(0.01)
"""
    )

    with pytest.raises(RuntimeError, match="did not write profile_export_aiperf.json"):
        run_profile(
            [sys.executable, str(fake), str(child_pid), str(worker), str(heartbeat)],
            tmp_path / "aiperf.log",
            tmp_path / "artifacts",
            timeout_seconds=10,
        )

    assert child_pid.read_text().isdigit()
    _assert_heartbeat_stopped(heartbeat)


@pytest.mark.parametrize(
    ("error_type", "log_message", "recovers"),
    [
        ("TimeoutError", _ALL_FAILED, True),
        ("ClientConnectorError", _ALL_FAILED, False),
        ("TimeoutError", "worker crashed", False),
    ],
)
def test_run_profile_recovers_only_expected_timeouts(
    tmp_path: Path, error_type: str, log_message: str, recovers: bool
) -> None:
    artifact_dir = tmp_path / "artifacts"
    artifact_dir.mkdir(parents=True)
    record = json.dumps({"error": {"type": error_type}})
    (artifact_dir / "profile_export.jsonl").write_text(f"{record}\n{record}\n")
    command = [sys.executable, "-c", f"print({log_message!r}); raise SystemExit(1)"]

    if recovers:
        export = run_profile(
            command,
            tmp_path / "aiperf.log",
            artifact_dir,
            timeout_seconds=10,
            expected_timeout_count=2,
        )
        summary = json.loads(export.read_text())
        assert summary["error_request_count"]["avg"] == 2
        assert summary["request_count"]["avg"] == 0
        assert summary["request_throughput"]["avg"] == 0.0
        return

    with pytest.raises(RuntimeError, match="AIPerf failed with status 1"):
        run_profile(
            command,
            tmp_path / "aiperf.log",
            artifact_dir,
            timeout_seconds=10,
            expected_timeout_count=2,
        )


def test_process_group_probe_ignores_an_unowned_reused_group(monkeypatch) -> None:
    def deny_signal(_process_group, _signal) -> None:
        raise PermissionError

    monkeypatch.setattr(aiperf_runner.os, "killpg", deny_signal)

    assert not aiperf_runner._process_group_exists(1234)


def test_validate_aiperf_version_rejects_uncovered_release(tmp_path) -> None:
    fake = tmp_path / "aiperf"
    fake.write_text("#!/bin/sh\nprintf '0.12.0\\n'\n")
    fake.chmod(0o755)

    with pytest.raises(RuntimeError, match="AIPerf 0.12.0 is unsupported"):
        validate_aiperf_version(str(fake))


def test_aggregate_exports_combines_independent_runs(tmp_path) -> None:
    exports = []
    for index, throughput in enumerate((10.0, 14.0), start=1):
        path = tmp_path / f"run-{index}.json"
        document = {
            "aiperf_version": "0.11.0",
            "error_request_count": None,
            "request_count": {"unit": "requests", "avg": 20},
            "request_throughput": {
                "unit": "requests/sec",
                "avg": throughput,
            },
            "time_to_first_token": {"unit": "ms", "p50": 10.0},
        }
        if index == 2:
            del document["error_request_count"]
            document["error_summary"] = []
        path.write_text(json.dumps(document))
        exports.append(path)

    aggregate_path = aggregate_exports(exports, tmp_path / "aggregate" / "report.json")
    metrics = json.loads(aggregate_path.read_text())["metrics"]

    assert metrics["request_throughput_avg"]["mean"] == 12.0
    assert metrics["request_throughput_avg"]["ci_low"] < 0
    assert metrics["error_request_count_avg"]["mean"] == 0.0

    mismatched = json.loads(exports[1].read_text())
    mismatched["time_to_first_token"]["unit"] = "seconds"
    exports[1].write_text(json.dumps(mismatched))
    with pytest.raises(RuntimeError, match="disagree on the unit"):
        aggregate_exports(exports, tmp_path / "invalid" / "report.json")

    mismatched["time_to_first_token"]["unit"] = "ms"
    mismatched["request_count"]["avg"] = True
    exports[1].write_text(json.dumps(mismatched))
    with pytest.raises(RuntimeError, match="has no numeric request_count.avg"):
        aggregate_exports(exports, tmp_path / "malformed" / "report.json")