Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(benchmark): report expected cancellation runs (#766)
Alex Fournier64def5647b71
diff --git a/scripts/aiperf_runner.py b/scripts/aiperf_runner.pyindex f83165df1..a7ebe2465 100644--- a/scripts/aiperf_runner.py+++ b/scripts/aiperf_runner.py@@ -31,6 +31,7 @@ _REQUIRED_STATISTICS = (("request_count", "avg"),("request_throughput", "avg"),)+_ALL_REQUESTS_FAILED = "inference request(s) failed; no successful responses were collected."def validate_aiperf_version(binary: str) -> None:@@ -98,11 +99,48 @@ def _stop_process_group(process: subprocess.Popen[bytes]) -> None:raise RuntimeError(f"could not reap AIPerf process {process.pid}") from error+def _recover_all_timeout_export(+ log_path: Path, artifact_dir: Path, expected_timeout_count: int+) -> Path | None:+ """Build the summary AIPerf omits when every expected request times out."""+ try:+ expected_log = f"All {expected_timeout_count} {_ALL_REQUESTS_FAILED}"+ with log_path.open(encoding="utf-8", errors="replace") as log:+ if not any(expected_log in line for line in log):+ return None+ timeout_count = 0+ with (artifact_dir / "profile_export.jsonl").open(encoding="utf-8") as records:+ for line in records:+ record = json.loads(line)+ error = record.get("error") if isinstance(record, dict) else None+ if not isinstance(error, dict) or error.get("type") != "TimeoutError":+ return None+ timeout_count += 1+ except (OSError, json.JSONDecodeError):+ return None+ if timeout_count != expected_timeout_count:+ return None+ export_path = artifact_dir / "profile_export_aiperf.json"+ summary = {+ "aiperf_version": SUPPORTED_AIPERF_VERSION,+ "error_request_count": {"unit": "requests", "avg": timeout_count},+ "request_count": {"unit": "requests", "avg": 0},+ "request_throughput": {"unit": "requests/sec", "avg": 0.0},+ }+ export_path.write_text(+ f"{json.dumps(summary, indent=2)}\n",+ encoding="utf-8",+ )+ return export_path++def run_profile(command: Sequence[str],log_path: Path,artifact_dir: Path,timeout_seconds: int,+ *,+ expected_timeout_count: int | None = None,) -> Path:"""Run one bounded AIPerf process and return its verified export."""artifact_dir.parent.mkdir(parents=True, exist_ok=True)@@ -130,6 +168,10 @@ def run_profile(finally:if not stopped and (process.poll() is None or _process_group_exists(process.pid)):_stop_process_group(process)+ if returncode == 1 and expected_timeout_count is not None:+ recovered = _recover_all_timeout_export(log_path, artifact_dir, expected_timeout_count)+ if recovered is not None:+ return recoveredif returncode != 0:raise RuntimeError(f"AIPerf failed with status {returncode}; see {log_path}")export_path = artifact_dir / "profile_export_aiperf.json"diff --git a/scripts/benchmark_routing_algorithms.py b/scripts/benchmark_routing_algorithms.pyindex 9a0a8baa7..35ba9fd4a 100755--- a/scripts/benchmark_routing_algorithms.py+++ b/scripts/benchmark_routing_algorithms.py@@ -1317,6 +1317,9 @@ def run_aiperf(output_dir / f"{run_label}.log",trial_root,aiperf_timeout_seconds(config, scenario, profile, concurrency),+ expected_timeout_count=(+ config.request_count if scenario.id == "client-cancellation" else None+ ),))if len(exports) == 1:diff --git a/tests/test_aiperf_runner.py b/tests/test_aiperf_runner.pyindex 71f5fafa7..e5d3bb4b4 100644--- a/tests/test_aiperf_runner.py+++ b/tests/test_aiperf_runner.py@@ -4,12 +4,15 @@import jsonimport sysimport time+from pathlib import Pathimport pytestimport scripts.aiperf_runner as aiperf_runnerfrom scripts.aiperf_runner import aggregate_exports, run_profile, validate_aiperf_version+_ALL_FAILED = "All 2 inference request(s) failed; no successful responses were collected."+def _write_stubborn_worker(worker) -> None:worker.write_text(@@ -99,6 +102,47 @@ while not Path(sys.argv[3]).exists():_assert_heartbeat_stopped(heartbeat)+ ("error_type", "log_message", "recovers"),+ [+ ("TimeoutError", _ALL_FAILED, True),+ ("ClientConnectorError", _ALL_FAILED, False),+ ("TimeoutError", "worker crashed", False),+ ],+)+def test_run_profile_recovers_only_expected_timeouts(+ tmp_path: Path, error_type: str, log_message: str, recovers: bool+) -> None:+ artifact_dir = tmp_path / "artifacts"+ artifact_dir.mkdir(parents=True)+ record = json.dumps({"error": {"type": error_type}})+ (artifact_dir / "profile_export.jsonl").write_text(f"{record}\n{record}\n")+ command = [sys.executable, "-c", f"print({log_message!r}); raise SystemExit(1)"]++ if recovers:+ export = run_profile(+ command,+ tmp_path / "aiperf.log",+ artifact_dir,+ timeout_seconds=10,+ expected_timeout_count=2,+ )+ summary = json.loads(export.read_text())+ assert summary["error_request_count"]["avg"] == 2+ assert summary["request_count"]["avg"] == 0+ assert summary["request_throughput"]["avg"] == 0.0+ return++ with pytest.raises(RuntimeError, match="AIPerf failed with status 1"):+ run_profile(+ command,+ tmp_path / "aiperf.log",+ artifact_dir,+ timeout_seconds=10,+ expected_timeout_count=2,+ )++def test_process_group_probe_ignores_an_unowned_reused_group(monkeypatch) -> None:def deny_signal(_process_group, _signal) -> None:raise PermissionErrordiff --git a/tests/test_routing_performance_report.py b/tests/test_routing_performance_report.pyindex 445357cc1..e58f00bea 100644--- a/tests/test_routing_performance_report.py+++ b/tests/test_routing_performance_report.py@@ -3,6 +3,7 @@import csvimport json+from collections.abc import Sequencefrom dataclasses import replacefrom pathlib import Path@@ -359,7 +360,11 @@ def test_aiperf_cells_use_disjoint_artifacts(tmp_path, monkeypatch) -> None:observed: list[tuple[Path, Path]] = []def fake_run_profile(- _command, log_path: Path, artifact_dir: Path, _timeout_seconds: int+ _command: Sequence[str],+ log_path: Path,+ artifact_dir: Path,+ _timeout_seconds: int,+ **_kwargs: object,) -> Path:observed.append((log_path, artifact_dir))artifact_dir.mkdir(parents=True)