Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
#![doc = include_str!("../README.md")]
mod client;
mod report;
mod scenarios;
mod stats;
use std::fs;
use std::future::Future;
use std::path::{Path, PathBuf};
use std::process::ExitCode;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
use clap::Parser;
use parking_lot::Mutex;
use reqwest::Client;
use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue};
use serde_json::json;
use tokio::sync::Notify;
use crate::client::{Endpoint, preflight, send_request};
use crate::report::{ResultsWriter, invalid_request_canary, reporter};
use crate::scenarios::{Scenario, ScenarioOptions, ScenarioSet};
use crate::stats::{RunStats, build_summary, now_utc_string, round3, utc_dir_stamp};
/// Command-line arguments for the soak test.
#[derive(Parser)]
#[command(
name = "switchyard-soak",
about = "Run a sustained, closed-loop load test against a live Switchyard server",
after_long_help = "Examples:\n switchyard-soak --model switchyard/general --duration 5m --concurrency 4\n switchyard-soak --model switchyard/general --scenario tool-call-burst --duration 5m\n switchyard-soak --model switchyard/general --export-scenarios scenarios\n\nThis command does not require oha or AIPerf. scripts/benchmark_routing_algorithms.py uses the exported scenario files with both tools.",
version
)]
pub struct Args {
/// HTTP base URL of the Switchyard server under test.
#[arg(long, default_value = "http://127.0.0.1:4000")]
base_url: String,
/// Exact route id advertised by GET /v1/models.
#[arg(long)]
model: String,
/// Time to keep sending load; use an s, m, or h suffix.
#[arg(long, value_parser = stats::parse_duration, default_value = "48h")]
duration: f64,
/// Requests kept in flight; each worker sends its next request after the last one ends.
#[arg(long, default_value_t = 16)]
concurrency: usize,
/// Default output-token limit; decode-heavy uses bounded 512 and 1,024 limits.
#[arg(long, default_value_t = 32)]
max_output_tokens: u32,
/// Bytes of repeated prefix added to each prompt to exercise request memory and caching.
#[arg(long, default_value_t = 1024)]
prompt_bytes: usize,
/// Named scenario collection; explicit --scenario values take precedence.
#[arg(long, value_enum, default_value_t = ScenarioSet::Standard)]
scenario_set: ScenarioSet,
/// Exact scenario id to run; repeat to select more than one scenario.
#[arg(long, action = clap::ArgAction::Append)]
scenario: Vec<String>,
/// Context-window size used to bound generated long-context scenarios.
#[arg(long, default_value_t = 32_768)]
context_window_tokens: usize,
/// Export selected AIPerf input files and exit without contacting a server.
#[arg(long)]
export_scenarios: Option<PathBuf>,
/// Seconds allowed to connect or wait between response bytes; an active stream may run longer.
#[arg(long, default_value_t = 120.0)]
request_timeout: f64,
/// Seconds between health, metrics, process, and result samples.
#[arg(long, default_value_t = 60.0)]
report_interval: f64,
/// Seconds between malformed-request checks; 0 turns this check off.
#[arg(long, default_value_t = 300.0)]
invalid_canary_interval: f64,
/// Largest passing request-error fraction, from 0 (none) through 1 (all).
#[arg(long, default_value_t = 0.0)]
max_error_rate: f64,
/// PID of a local switchyard-server process whose RSS and CPU should be sampled.
#[arg(long)]
server_pid: Option<u32>,
/// Largest passing first-to-last RSS increase in MiB; requires --server-pid.
#[arg(long, requires = "server_pid")]
max_rss_growth_mib: Option<f64>,
/// Name of an environment variable that holds the endpoint's bearer token.
#[arg(long)]
api_key_env: Option<String>,
/// New directory to create for config, interval, error, and summary files.
#[arg(long)]
results_dir: Option<PathBuf>,
}
impl Args {
/// Reject invalid numeric combinations after clap parses their types.
pub fn validate(&self) -> Result<(), String> {
if self.concurrency == 0 {
return Err("--concurrency must be greater than zero".to_string());
}
if self.max_output_tokens == 0 || self.prompt_bytes == 0 {
return Err(
"--max-output-tokens and --prompt-bytes must be greater than zero".to_string(),
);
}
ScenarioOptions {
model: &self.model,
prompt_bytes: self.prompt_bytes,
max_output_tokens: self.max_output_tokens,
context_window_tokens: self.context_window_tokens,
}
.validate()?;
if !self.request_timeout.is_finite()
|| !self.report_interval.is_finite()
|| self.request_timeout <= 0.0
|| self.report_interval <= 0.0
{
return Err(
"--request-timeout and --report-interval must be greater than zero".to_string(),
);
}
if !self.invalid_canary_interval.is_finite() || self.invalid_canary_interval < 0.0 {
return Err("--invalid-canary-interval must be zero or greater".to_string());
}
if !(0.0..=1.0).contains(&self.max_error_rate) {
return Err("--max-error-rate must be between 0 and 1".to_string());
}
if self.server_pid == Some(0) {
return Err("--server-pid must be greater than zero".to_string());
}
if self
.max_rss_growth_mib
.is_some_and(|growth| !growth.is_finite() || growth < 0.0)
{
return Err("--max-rss-growth-mib must be zero or greater".to_string());
}
Ok(())
}
}
#[derive(Clone)]
struct RunContext {
client: Client,
base_url: String,
stop: Arc<Stop>,
stats: Arc<Mutex<RunStats>>,
writer: Arc<Mutex<ResultsWriter>>,
}
struct Workload {
model: String,
scenarios: Vec<Scenario>,
}
/// A one-shot stop signal that many tasks can wait on and any task can raise.
struct Stop {
flag: AtomicBool,
notify: Notify,
}
impl Stop {
fn new() -> Self {
Self {
flag: AtomicBool::new(false),
notify: Notify::new(),
}
}
fn set(&self) {
self.flag.store(true, Ordering::SeqCst);
self.notify.notify_waiters();
}
fn is_set(&self) -> bool {
self.flag.load(Ordering::SeqCst)
}
/// Resolve once the signal is raised, now or later.
async fn wait(&self) {
loop {
if self.is_set() {
return;
}
// Register before the second flag check so a set() between the two still wakes us.
let notified = self.notify.notified();
tokio::pin!(notified);
notified.as_mut().enable();
if self.is_set() {
return;
}
notified.await;
}
}
}
/// Send closed-loop traffic until the run stops.
async fn worker(
context: RunContext,
workload: Arc<Workload>,
worker_id: usize,
) -> Result<(), String> {
let mut request_number = 0;
while !context.stop.is_set() {
let scenario = &workload.scenarios[request_number % workload.scenarios.len()];
let stream = (request_number / workload.scenarios.len()).is_multiple_of(2);
let request = scenario.request(request_number / workload.scenarios.len(), stream);
let session_id = format!("{}-worker-{worker_id}", request.session_id);
let started = Instant::now();
let result = send_request(
&context.client,
&context.base_url,
request.endpoint,
&session_id,
request.body,
)
.await;
let latency_ms = started.elapsed().as_secs_f64() * 1000.0;
context.stats.lock().record(
request.endpoint.as_str(),
scenario.id,
latency_ms,
result.as_ref().err().map(|error| error.kind.as_str()),
);
if let Err(error) = result {
context
.writer
.lock()
.write_error(&json!({
"timestamp_utc": now_utc_string(),
"worker": worker_id,
"scenario": scenario.id,
"endpoint": request.endpoint.as_str(),
"stream": stream,
"latency_ms": round3(latency_ms),
"error": error.kind,
"detail": error.detail,
}))
.map_err(|error| error.to_string())?;
}
request_number = request_number.wrapping_add(1);
}
Ok(())
}
/// Stop the run when a background task fails or panics.
async fn guard_stop(
stop: Arc<Stop>,
task: impl Future<Output = Result<(), String>>,
) -> Result<(), String> {
struct StopOnDrop(Arc<Stop>, bool);
impl Drop for StopOnDrop {
fn drop(&mut self) {
if self.1 {
self.0.set();
}
}
}
let mut guard = StopOnDrop(stop, true);
let result = task.await;
guard.1 = result.is_err();
result
}
/// Add one joined task's error to the failure reasons; cancelled tasks are expected.
fn collect_failure(
name: &str,
result: Result<Result<(), String>, tokio::task::JoinError>,
failures: &mut Vec<String>,
) {
match result {
Ok(Ok(())) => {}
Ok(Err(reason)) => failures.push(format!("{name} failed: {reason}")),
Err(join) if join.is_cancelled() => {}
Err(join) => failures.push(format!("{name} failed: {join}")),
}
}
/// Raise *stop* on SIGINT or SIGTERM so an operator can end the run cleanly.
fn spawn_signal_listener(stop: Arc<Stop>) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
#[cfg(unix)]
{
use tokio::signal::unix::{SignalKind, signal};
let mut interrupt = signal(SignalKind::interrupt()).ok();
let mut terminate = signal(SignalKind::terminate()).ok();
let wait_interrupt = async {
match interrupt.as_mut() {
Some(stream) => {
stream.recv().await;
}
None => std::future::pending::<()>().await,
}
};
let wait_terminate = async {
match terminate.as_mut() {
Some(stream) => {
stream.recv().await;
}
None => std::future::pending::<()>().await,
}
};
tokio::select! {
_ = wait_interrupt => {}
_ = wait_terminate => {}
}
stop.set();
}
#[cfg(not(unix))]
{
let _ = tokio::signal::ctrl_c().await;
stop.set();
}
})
}
/// Record every non-secret input with the normalized duration and fixed request variants.
fn write_config(
results_dir: &Path,
args: &Args,
model: &str,
scenarios: &[Scenario],
) -> Result<(), String> {
let config = json!({
"base_url": args.base_url,
"model": model,
"duration_seconds": args.duration,
"concurrency": args.concurrency,
"endpoints": Endpoint::ALL.map(Endpoint::as_str),
"streaming": [true, false],
"max_output_tokens": args.max_output_tokens,
"prompt_bytes": args.prompt_bytes,
"context_window_tokens": args.context_window_tokens,
"scenarios": scenarios.iter().map(|scenario| scenario.id).collect::<Vec<_>>(),
"request_timeout": args.request_timeout,
"report_interval": args.report_interval,
"invalid_canary_interval": args.invalid_canary_interval,
"max_error_rate": args.max_error_rate,
"server_pid": args.server_pid,
"max_rss_growth_mib": args.max_rss_growth_mib,
"api_key_env": args.api_key_env,
});
let body = serde_json::to_string_pretty(&config).map_err(|error| error.to_string())?;
fs::write(results_dir.join("config.json"), format!("{body}\n"))
.map_err(|error| error.to_string())
}
/// Run the configured soak test and return a process exit code (0 pass, 1 fail).
pub async fn run(args: Args) -> Result<i32, String> {
args.validate()?;
let scenarios = scenarios::select(
ScenarioOptions {
model: &args.model,
prompt_bytes: args.prompt_bytes,
max_output_tokens: args.max_output_tokens,
context_window_tokens: args.context_window_tokens,
},
args.scenario_set,
&args.scenario,
)?;
if let Some(output_dir) = &args.export_scenarios {
let manifest = scenarios::export(&scenarios, &args.model, output_dir)?;
println!("Exported scenario manifest: {}", manifest.display());
return Ok(0);
}
let token = match &args.api_key_env {
Some(var) => {
let token = std::env::var(var).ok().filter(|value| !value.is_empty());
if token.is_none() {
return Err(format!("${var} is not set"));
}
token
}
None => None,
};
// Per-operation timeouts, not a whole-request deadline: a healthy long stream that keeps
// delivering bytes must not be aborted, so bound connect time and idle time between reads.
let request_timeout = Duration::from_secs_f64(args.request_timeout);
let mut builder = Client::builder()
.no_proxy()
.connect_timeout(request_timeout)
.read_timeout(request_timeout)
.pool_max_idle_per_host(args.concurrency + 4);
if let Some(token) = &token {
let mut headers = HeaderMap::new();
let value = HeaderValue::from_str(&format!("Bearer {token}"))
.map_err(|_| "the bearer token is not a valid HTTP header value".to_string())?;
headers.insert(AUTHORIZATION, value);
builder = builder.default_headers(headers);
}
let client = builder.build().map_err(|error| error.to_string())?;
let base_url = args.base_url.trim_end_matches('/').to_string();
preflight(&client, &base_url, &args.model).await?;
let results_dir = args
.results_dir
.clone()
.unwrap_or_else(|| PathBuf::from("soak-results").join(utc_dir_stamp()));
let writer = Arc::new(Mutex::new(
ResultsWriter::new(&results_dir).map_err(|error| error.to_string())?,
));
write_config(&results_dir, &args, &args.model, &scenarios)?;
println!(
"Soak started: model={} duration={}s concurrency={} scenarios={} results={}",
args.model,
args.duration,
args.concurrency,
scenarios
.iter()
.map(|scenario| scenario.id)
.collect::<Vec<_>>()
.join(","),
results_dir.display(),
);
let started = Instant::now();
let stats = Arc::new(Mutex::new(RunStats::new(2026)));
let stop = Arc::new(Stop::new());
let workers_done = Arc::new(Stop::new());
let context = RunContext {
client,
base_url,
stop: stop.clone(),
stats: stats.clone(),
writer: writer.clone(),
};
let signal_listener = spawn_signal_listener(stop.clone());
let deadline = {
let stop = stop.clone();
let stats = stats.clone();
let duration = Duration::from_secs_f64(args.duration);
tokio::spawn(async move {
tokio::time::sleep(duration).await;
stats.lock().completed_duration = true;
stop.set();
})
};
let workload = Arc::new(Workload {
model: args.model.clone(),
scenarios,
});
let mut worker_handles = Vec::new();
for worker_id in 0..args.concurrency {
let handle = tokio::spawn(guard_stop(
stop.clone(),
worker(context.clone(), workload.clone(), worker_id),
));
worker_handles.push((format!("worker-{worker_id}"), handle));
}
let reporter_handle = tokio::spawn(guard_stop(
stop.clone(),
reporter(
context.clone(),
started,
Duration::from_secs_f64(args.report_interval),
args.duration,
args.server_pid,
workers_done.clone(),
),
));
let canary_handle = tokio::spawn(guard_stop(
stop.clone(),
invalid_request_canary(context, args.invalid_canary_interval, workload),
));
stop.wait().await;
deadline.abort();
signal_listener.abort();
// A crashed worker/reporter/canary must not discard the run: record it as a failure reason
// so the summary is still written and the run fails closed.
let mut task_failures = Vec::new();
for (name, handle) in worker_handles {
collect_failure(&name, handle.await, &mut task_failures);
}
workers_done.set();
collect_failure("reporter", reporter_handle.await, &mut task_failures);
collect_failure(
"invalid-request-canary",
canary_handle.await,
&mut task_failures,
);
let elapsed = started.elapsed().as_secs_f64();
let (error_records, dropped_error_records) = { writer.lock().error_counts() };
let summary = build_summary(
&stats.lock(),
elapsed,
args.max_error_rate,
args.max_rss_growth_mib,
error_records,
dropped_error_records,
&task_failures,
);
let summary_path = results_dir.join("summary.json");
let summary_body = serde_json::to_string_pretty(&summary).map_err(|error| error.to_string())?;
fs::write(&summary_path, format!("{summary_body}\n")).map_err(|error| error.to_string())?;
let label = if summary.passed { "PASS" } else { "FAIL" };
println!(
"Soak {label}: requests={} error_rate={:.4}% p95_ms={} summary={}",
summary.requests,
summary.error_rate * 100.0,
summary
.latency_p95_ms
.map(|value| value.to_string())
.unwrap_or_default(),
summary_path.display(),
);
for reason in &summary.failure_reasons {
println!("- {reason}");
}
Ok(if summary.passed { 0 } else { 1 })
}
/// Parse arguments, run the test on a multi-thread runtime, and map the result to an exit code.
pub fn cli_main() -> ExitCode {
let args = Args::parse();
if let Err(message) = args.validate() {
eprintln!("switchyard-soak: {message}");
return ExitCode::from(2);
}
let runtime = match tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
{
Ok(runtime) => runtime,
Err(error) => {
eprintln!("soak test setup failed: {error}");
return ExitCode::from(2);
}
};
match runtime.block_on(run(args)) {
Ok(0) => ExitCode::SUCCESS,
Ok(_) => ExitCode::from(1),
Err(error) => {
eprintln!("soak test setup failed: {error}");
ExitCode::from(2)
}
}
}