Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(operations): add release soak and routing benchmarks (#176)

Elyas Mehtabuddin71eb274db6d6
diff --git a/.gitignore b/.gitignore
index fdc44128c..c456be335 100644
--- a/.gitignore
+++ b/.gitignore
@@ -150,6 +150,9 @@ plans/
# Benchmark result directories
bench_results*/
+soak-results/
+local-soak-test-results/
+routing-benchmark-results/
jobs/**/*.*
pytest-of-*/
diff --git a/Cargo.lock b/Cargo.lock
index 405dda9da..ff7a91185 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2388,6 +2388,23 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "switchyard-soak"
+version = "0.2.0"
+dependencies = [
+ "axum",
+ "clap",
+ "futures-util",
+ "humantime",
+ "parking_lot",
+ "rand 0.10.2",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "tempfile",
+ "tokio",
+]
+
[[package]]
name = "switchyard-translation"
version = "0.2.0"
diff --git a/Cargo.toml b/Cargo.toml
index 98433d8c1..d929d0c58 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,6 +10,7 @@ members = [
"crates/protocol",
"crates/switchyard-server",
"crates/switchyard-skill-distillation",
+ "crates/switchyard-soak",
"crates/switchyard-translation",
]
diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rs
index 95f4c7212..8a9e42c47 100644
--- a/crates/libsy/src/algorithms/noop.rs
+++ b/crates/libsy/src/algorithms/noop.rs
@@ -23,11 +23,12 @@ impl Algorithm for Noop {
}
async fn route(self: Arc<Self>, _driver: Driver, request: Request) -> Result<RoutingOutcome> {
+ let streaming = request.llm_request.stream;
let model_id = request
.model_id()
.unwrap_or_else(|| ModelId::from("switchyard/noop"));
tracing::info!(target = %model_id, "noop returned its synthetic response");
- let llm_response = LlmResponse::Agg(AggLlmResponse {
+ let aggregate = AggLlmResponse {
id: Some("switchyard-noop".to_string()),
model: Some(model_id.to_string()),
outputs: vec![ResponseOutput {
@@ -38,7 +39,12 @@ impl Algorithm for Noop {
stop_reason: Some(StopReason::EndTurn),
}],
..Default::default()
- });
+ };
+ let llm_response = if streaming {
+ LlmResponse::Stream(aggregate.into_stream())
+ } else {
+ LlmResponse::Agg(aggregate)
+ };
let response = Response {
llm_response,
metadata: request.metadata.clone(),
@@ -75,4 +81,24 @@ mod tests {
assert_eq!(response.selected_model(), Some(TEST_MODEL));
Ok(())
}
+
+ #[tokio::test]
+ async fn test_noop_algo_streams_when_requested() -> Result<()> {
+ let request = Request {
+ llm_request: LlmRequest {
+ model: Some("streaming-noop".to_string()),
+ messages: vec![Message::text(Role::User, "hi")],
+ stream: true,
+ ..LlmRequest::default()
+ },
+ raw_request: None,
+ metadata: None,
+ };
+
+ let algorithm: Arc<dyn Algorithm> = Arc::new(Noop {});
+ let (_, response) = test_drive(algorithm, request, echo()).await?;
+
+ assert!(matches!(response.llm_response, LlmResponse::Stream(_)));
+ Ok(())
+ }
}
diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs
index 420ac5e2b..0238bd041 100644
--- a/crates/switchyard-server/src/config.rs
+++ b/crates/switchyard-server/src/config.rs
@@ -1676,6 +1676,21 @@ target = "azure"
return Err(ServerError::new("primary llm client is missing"));
};
assert_eq!(primary.max_retries, 0);
+
+ let maximum = VALID_CONFIG.replacen(
+ "base_url = \"https://example.test/v1\"",
+ &format!(
+ "base_url = \"https://example.test/v1\"\nmax_retries = {MAX_CONFIGURED_RETRIES}"
+ ),
+ 1,
+ );
+ let config: ServerConfig = toml::from_str(&maximum).map_err(|error| {
+ ServerError::new(format!("failed to parse maximum retry config: {error}"))
+ })?;
+ let Some(primary) = config.llm_clients.get("primary") else {
+ return Err(ServerError::new("primary llm client is missing"));
+ };
+ assert_eq!(primary.max_retries, MAX_CONFIGURED_RETRIES);
Ok(())
}
diff --git a/crates/switchyard-soak/Cargo.toml b/crates/switchyard-soak/Cargo.toml
new file mode 100644
index 000000000..242dbed17
--- /dev/null
+++ b/crates/switchyard-soak/Cargo.toml
@@ -0,0 +1,37 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+[package]
+name = "switchyard-soak"
+version.workspace = true
+description = "Sustained load tester for a live Switchyard server"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+readme = "README.md"
+
+[[bin]]
+name = "switchyard-soak"
+path = "src/main.rs"
+
+[[example]]
+name = "switchyard-soak-mock"
+path = "examples/mock_server.rs"
+
+[dependencies]
+clap = { version = "4", features = ["derive"] }
+futures-util.workspace = true
+humantime = "2.4"
+parking_lot.workspace = true
+rand.workspace = true
+reqwest.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+tokio.workspace = true
+
+[dev-dependencies]
+axum = "0.8"
+tempfile = "3"
+tokio.workspace = true
diff --git a/crates/switchyard-soak/README.md b/crates/switchyard-soak/README.md
new file mode 100644
index 000000000..7f739f52f
--- /dev/null
+++ b/crates/switchyard-soak/README.md
@@ -0,0 +1,119 @@
+# switchyard-soak
+
+`switchyard-soak` keeps a fixed number of requests in flight against one route on a live
+`switchyard-server`. Its scenario catalog covers short and long inputs, long outputs, shared
+prefixes, mixed traffic, growing conversations, tool schemas, tool-call bursts, routing signals,
+and bounded failure cases. The short baseline also exercises Chat Completions, Messages, and
+Responses in streaming and non-streaming form. The command samples health, metrics, and an
+optional local server process, then exits with status 1 when a release gate fails.
+
+Build the server and soak tester from the commit being tested:
+
+```bash
+cargo build --release -p switchyard-server -p switchyard-soak \
+ --bins --example switchyard-soak-mock
+```
+
+Run a five-minute check against a route advertised by `GET /v1/models`:
+
+```bash
+target/release/switchyard-soak \
+ --base-url http://127.0.0.1:4000 \
+ --model switchyard/general \
+ --duration 5m \
+ --concurrency 4 \
+ --report-interval 10
+```
+
+Run the 48-hour release gate and measure the local server process:
+
+```bash
+target/release/switchyard-soak \
+ --model switchyard/general \
+ --duration 48h \
+ --concurrency 16 \
+ --server-pid "$SWITCHYARD_SERVER_PID" \
+ --max-rss-growth-mib 512
+```
+
+If the endpoint needs a bearer token, put the secret in an environment variable and pass the
+variable's name:
+
+```bash
+export SWITCHYARD_SOAK_TOKEN="..."
+target/release/switchyard-soak \
+ --model switchyard/general \
+ --api-key-env SWITCHYARD_SOAK_TOKEN
+```
+
+## Flags
+
+| Flag | Default | What it does |
+|---|---:|---|
+| `--base-url URL` | `http://127.0.0.1:4000` | Sends every check and inference request to this Switchyard server. |
+| `--model ID` | required | Selects one exact route id returned by `GET /v1/models`. The test stops before sending load if the id is missing. |
+| `--duration TIME` | `48h` | Keeps the load running for this long. Use seconds, minutes, or hours, such as `30s`, `5m`, or `48h`. |
+| `--concurrency N` | `16` | Keeps this many requests in flight. Each worker waits for its response before sending another request. |
+| `--max-output-tokens N` | `32` | Sets the default output limit. `decode-heavy` uses its bounded 512-token and 1,024-token limits. |
+| `--prompt-bytes N` | `1024` | Sizes the repeated input used by short and prefix-reuse scenarios. Larger values put more pressure on request memory and prefix caching. |
+| `--scenario-set NAME` | `standard` | Selects `core`, `agentic`, `resilience`, `standard`, or `all`. `standard` includes core and agentic scenarios without expected failures. |
+| `--scenario ID` | none | Selects one exact scenario id. Repeat the flag to select more scenarios; explicit ids take precedence over `--scenario-set`. |
+| `--context-window-tokens N` | `32768` | Bounds generated long-context and overflow inputs. The command rejects values above 1,000,000. |
+| `--export-scenarios PATH` | none | Writes a manifest and AIPerf `inputs-json` files, then exits without contacting a server. The performance script uses this mode. |
+| `--request-timeout SECONDS` | `120` | Limits connection time and idle time between response bytes. A healthy stream may run longer if it keeps producing data. |
+| `--report-interval SECONDS` | `60` | Sets how often the test samples health, metrics, RSS, CPU, and interval latency. |
+| `--invalid-canary-interval SECONDS` | `300` | Sends malformed input this often, expects HTTP 400, and then checks recovery. Set it to `0` only when a permissive mock cannot reject the canary. |
+| `--max-error-rate FRACTION` | `0` | Allows at most this fraction of inference requests to fail. `0.01` means one percent. The value must be between 0 and 1. |
+| `--server-pid PID` | none | Samples RSS and CPU from this local `switchyard-server` process. Omit it for a remote server. |
+| `--max-rss-growth-mib MIB` | none | Fails when the last RSS sample exceeds the first by more than this amount. It requires `--server-pid`. |
+| `--api-key-env NAME` | none | Reads a bearer token from this environment variable without putting the secret in the command or result files. |
+| `--results-dir PATH` | `soak-results/<UTC time>` | Creates this new directory for the run. The command refuses to reuse an existing directory. |
+| `--help` | n/a | Prints the command reference and examples. |
+| `--version` | n/a | Prints the crate version. |
+
+The scenario source lives in `src/scenarios/`. Each request shape has one file and a pure builder.
+The exported manifest is the only scenario input read by the Python performance runner, so model
+ids, sessions, tools, and message histories cannot drift between the Rust soak and AIPerf runs.
+Concurrency knees and traffic bursts are load profiles attached to the short baseline, not
+duplicate request scenarios.
+
+The catalog contains:
+
+| Group | Scenarios |
+|---|---|
+| Core | `short-interactive`, `long-context`, `decode-heavy`, `prefix-reuse`, `mixed-traffic` |
+| Agentic | `growing-conversation`, `large-tool-catalog`, `tool-call-burst`, `stage-transitions`, `classifier-mix` |
+| Resilience | `context-overflow`, `failure-pressure`, `client-cancellation` |
+
+## Check the local server and load tools
+
+`scripts/run_local_soak_test.py` starts the crate's request-aware scenario backend and
+`switchyard-server`, sends one HTTP request through each configured route, then runs `oha` and
+NVIDIA AIPerf sequentially for every routing algorithm. The backend returns valid classifier
+verdicts and injects the context, retry, stream, and cancellation cases named by resilience
+scenarios. It resets transient-failure counters before each AIPerf cell so every routing algorithm
+receives the same attempts. For streaming requests it emits the scenario's requested output length
+with a configurable delay between tokens, which keeps local TTFT, ITL, and token-throughput
+comparisons deterministic. The local test needs no provider key and incurs no inference cost.
+
+Run `scripts/benchmark_routing_algorithms.py` by itself against an existing Switchyard server to
+compare the same routes with real model output. oha runs only for the fixed `short-interactive`
+baseline. AIPerf replays every selected session and reports TTFT, ITL, request throughput, output
+token throughput, multi-run confidence, selected-target calls, classifier calls, and routing
+overhead. Add `--direct-base-url` and `--direct-model` to compare each route with the same backend
+without Switchyard. That comparison adds an annotated TTFT and output-token-throughput SVG to the
+Markdown report. The operations guide explains how to keep the comparison fair and when to use the
+local backend or real models.
+
+The script needs the built Rust programs plus installed `oha` and AIPerf commands.
+`switchyard-soak` itself does not require those extra programs and can run by itself against a
+release server. See the [operations guide](../../docs/operations/soak_test.md) for setup, the local
+command, the 48-hour release run, and result review.
+
+## Results
+
+Each run creates `config.json`, `intervals.csv`, `errors.jsonl`, and `summary.json`. Error details
+stop at 10,000 records, and run-wide latency uses a 100,000-sample reservoir, so failures and a
+high request rate cannot grow memory or disk use without bound.
+
+See `docs/operations/soak_test.md` for release preparation, pass criteria, and result review.
diff --git a/crates/switchyard-soak/examples/mock_server.rs b/crates/switchyard-soak/examples/mock_server.rs
new file mode 100644
index 000000000..c92578fd6
--- /dev/null
+++ b/crates/switchyard-soak/examples/mock_server.rs
@@ -0,0 +1,280 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Request-aware local backend used by `scripts/run_local_soak_test.py`.
+
+use std::collections::HashMap;
+use std::convert::Infallible;
+use std::process::ExitCode;
+use std::sync::Arc;
+use std::time::Duration;
+
+use axum::Router;
+use axum::extract::{Json, State};
+use axum::http::StatusCode;
+use axum::response::sse::{Event, Sse};
+use axum::response::{IntoResponse, Response};
+use axum::routing::{get, post};
+use clap::Parser;
+use futures_util::{StreamExt, stream};
+use parking_lot::Mutex;
+use serde_json::{Value, json};
+
+#[derive(Parser)]
+#[command(
+ name = "switchyard-soak-mock",
+ about = "Start the request-aware backend for the local Switchyard scenario test",
+ after_long_help = "Example:\n cargo run --release -p switchyard-soak --example switchyard-soak-mock -- --port 8100 --latency-ms 40",
+ version
+)]
+struct Args {
+ /// Local TCP port used by the backend.
+ #[arg(long, default_value_t = 8100)]
+ port: u16,
+
+ /// Artificial delay, in milliseconds, added to ordinary responses.
+ #[arg(long, default_value_t = 40)]
+ latency_ms: u64,
+
+ /// Artificial delay between streamed output tokens, in milliseconds.
+ #[arg(long, default_value_t = 1)]
+ token_latency_ms: u64,
+}
+
+#[derive(Clone)]
+struct BackendState {
+ latency: Duration,
+ token_latency: Duration,
+ attempts: Arc<Mutex<HashMap<String, u64>>>,
+}
+
+// AIPerf requires this OpenAI field; a fixed value keeps local runs deterministic.
+const FIXED_CREATED_AT: u64 = 1_700_000_000;
+
+fn scenario_marker(body: &Value) -> Option<&str> {
+ body.get("messages")?
+ .as_array()?
+ .iter()
+ .filter_map(|message| message.get("content").and_then(Value::as_str))
+ .find_map(|content| {
+ let marker = content.split_once("[scenario:")?.1;
+ marker.split_once(']').map(|(name, _rest)| name)
+ })
+}
+
+fn classifier_content(marker: Option<&str>) -> &'static str {
+ if marker == Some("classifier_invalid") {
+ return "not a JSON verdict";
+ }
+ if marker == Some("classifier_hard") {
+ return r#"{"crux":"distributed diagnosis","primary_rule":"LIM-1","capability_boundary":"unsupported","p_solve":0.1}"#;
+ }
+ r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":1.0}"#
+}
+
+fn completion(model: &str, content: &str) -> Value {
+ json!({
+ "id": "chatcmpl-switchyard-soak",
+ "object": "chat.completion",
+ "created": FIXED_CREATED_AT,
+ "model": model,
+ "choices": [{
+ "index": 0,
+ "message": {"role": "assistant", "content": content},
+ "finish_reason": "stop"
+ }],
+ "usage": {"prompt_tokens": 32, "completion_tokens": 2, "total_tokens": 34}
+ })
+}
+
+fn requested_output_tokens(body: &Value) -> u64 {
+ body.get("max_completion_tokens")
+ .or_else(|| body.get("max_tokens"))
+ .and_then(Value::as_u64)
+ .unwrap_or(2)
+ .clamp(1, 4_096)
+}
+
+fn stream(
+ model: &str,
+ truncated: bool,
+ completion_tokens: u64,
+ token_latency: Duration,
+) -> Response {
+ let mut events = Vec::with_capacity(completion_tokens as usize + 2);
+ for index in 0..completion_tokens {
+ events.push(
+ json!({
+ "id": "chatcmpl-switchyard-soak",
+ "object": "chat.completion.chunk",
+ "created": FIXED_CREATED_AT,
+ "model": model,
+ "choices": [{
+ "index": 0,
+ "delta": {
+ "role": (index == 0).then_some("assistant"),
+ "content": if index == 0 { "token" } else { " token" }
+ },
+ "finish_reason": null
+ }]
+ })
+ .to_string(),
+ );
+ if truncated {
+ break;
+ }
+ }
+ if !truncated {
+ events.push(
+ json!({
+ "id": "chatcmpl-switchyard-soak",
+ "object": "chat.completion.chunk",
+ "created": FIXED_CREATED_AT,
+ "model": model,
+ "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
+ "usage": {
+ "prompt_tokens": 32,
+ "completion_tokens": completion_tokens,
+ "total_tokens": 32 + completion_tokens
+ }
+ })
+ .to_string(),
+ );
+ events.push("[DONE]".to_string());
+ }
+ let events =
+ stream::iter(events.into_iter().enumerate()).then(move |(index, data)| async move {
+ if index > 0 {
+ tokio::time::sleep(token_latency).await;
+ }
+ Ok::<Event, Infallible>(Event::default().data(data))
+ });
+ Sse::new(events).into_response()
+}
+
+async fn chat(State(state): State<BackendState>, Json(body): Json<Value>) -> Response {
+ let model = body
+ .get("model")
+ .and_then(Value::as_str)
+ .unwrap_or_default();
+ let marker = scenario_marker(&body);
+ if marker == Some("client_cancellation") {
+ tokio::time::sleep(Duration::from_secs(2)).await;
+ } else {
+ tokio::time::sleep(state.latency).await;
+ }
+
+ if model == "mock/classifier" {
+ return Json(completion(model, classifier_content(marker))).into_response();
+ }
+ if model == "mock/weak" && marker == Some("context_overflow") {
+ return (
+ StatusCode::BAD_REQUEST,
+ Json(json!({
+ "error": {
+ "code": "context_length_exceeded",
+ "message": "the weak target context window is too small"
+ }
+ })),
+ )
+ .into_response();
+ }
+
+ for (name, status) in [
+ ("upstream_429", StatusCode::TOO_MANY_REQUESTS),
+ ("upstream_500", StatusCode::INTERNAL_SERVER_ERROR),
+ ] {
+ if marker == Some(name) {
+ let key = format!("{model}:{name}");
+ let attempt = {
+ let mut attempts = state.attempts.lock();
+ let attempt = attempts.entry(key).or_default();
+ *attempt += 1;
+ *attempt
+ };
+ if attempt <= 2 {
+ return (
+ status,
+ Json(
+ json!({"error": {"message": format!("injected {name} attempt {attempt}")}}),
+ ),
+ )
+ .into_response();
+ }
+ }
+ }
+
+ if body.get("stream") == Some(&Value::Bool(true)) {
+ return stream(
+ model,
+ marker == Some("truncated_stream"),
+ requested_output_tokens(&body),
+ state.token_latency,
+ );
+ }
+ Json(completion(model, "OK")).into_response()
+}
+
+async fn run(args: Args) -> Result<(), String> {
+ if args.port == 0 {
+ return Err("--port must be greater than zero".to_string());
+ }
+ let listener = tokio::net::TcpListener::bind(("127.0.0.1", args.port))
+ .await
+ .map_err(|error| error.to_string())?;
+ let address = listener.local_addr().map_err(|error| error.to_string())?;
+ let state = BackendState {
+ latency: Duration::from_millis(args.latency_ms),
+ token_latency: Duration::from_millis(args.token_latency_ms),
+ attempts: Arc::new(Mutex::new(HashMap::new())),
+ };
+ let app = Router::new()
+ .route("/health", get(|| async { Json(json!({"status": "ok"})) }))
+ .route(
+ "/reset",
+ post(|State(state): State<BackendState>| async move {
+ state.attempts.lock().clear();
+ Json(json!({"status": "reset"}))
+ }),
+ )
+ .route("/v1/chat/completions", post(chat))
+ .with_state(state);
+ println!("Scenario backend is ready at http://{address}");
+ axum::serve(listener, app)
+ .with_graceful_shutdown(async {
+ if let Err(error) = tokio::signal::ctrl_c().await {
+ eprintln!("could not register the shutdown signal: {error}");
+ std::future::pending::<()>().await;
+ }
+ })
+ .await
+ .map_err(|error| error.to_string())
+}
+
+#[cfg(test)]
+mod tests {
+ use serde_json::json;
+
+ use super::requested_output_tokens;
+
+ #[test]
+ fn output_tokens_follow_openai_limits_and_stay_bounded() {
+ assert_eq!(requested_output_tokens(&json!({"max_tokens": 512})), 512);
+ assert_eq!(
+ requested_output_tokens(&json!({"max_completion_tokens": 8_192})),
+ 4_096
+ );
+ assert_eq!(requested_output_tokens(&json!({})), 2);
+ }
+}
+
+#[tokio::main]
+async fn main() -> ExitCode {
+ match run(Args::parse()).await {
+ Ok(()) => ExitCode::SUCCESS,
+ Err(error) => {
+ eprintln!("switchyard-soak-mock failed: {error}");
+ ExitCode::FAILURE
+ }
+ }
+}
diff --git a/crates/switchyard-soak/src/client.rs b/crates/switchyard-soak/src/client.rs
new file mode 100644
index 000000000..5fe7074b1
--- /dev/null
+++ b/crates/switchyard-soak/src/client.rs
@@ -0,0 +1,525 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! HTTP against the public Switchyard APIs: preflight, one request, and server-state reads.
+
+use futures_util::StreamExt;
+use reqwest::Client;
+use reqwest::header::CONTENT_TYPE;
+use serde_json::{Value, json};
+
+const SERVER_REQUESTS_METRIC: &str = "switchyard_total_requests";
+const SERVER_ERRORS_METRIC: &str = "switchyard_total_errors";
+
+/// One public Switchyard API the soak test exercises.
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub enum Endpoint {
+ Chat,
+ Messages,
+ Responses,
+}
+
+impl Endpoint {
+ pub const ALL: [Endpoint; 3] = [Endpoint::Chat, Endpoint::Messages, Endpoint::Responses];
+
+ pub fn path(self) -> &'static str {
+ match self {
+ Endpoint::Chat => "/v1/chat/completions",
+ Endpoint::Messages => "/v1/messages",
+ Endpoint::Responses => "/v1/responses",
+ }
+ }
+
+ /// Field a successful response for this endpoint must contain.
+ fn required_field(self) -> &'static str {
+ match self {
+ Endpoint::Chat => "choices",
+ Endpoint::Messages => "content",
+ Endpoint::Responses => "output",
+ }
+ }
+
+ pub fn as_str(self) -> &'static str {
+ match self {
+ Endpoint::Chat => "chat",
+ Endpoint::Messages => "messages",
+ Endpoint::Responses => "responses",
+ }
+ }
+}
+
+/// Keep at most *limit* characters, on a char boundary, for a logged detail string.
+fn truncate(text: &str, limit: usize) -> String {
+ text.chars().take(limit).collect()
+}
+
+fn transport_error_kind(error: &reqwest::Error) -> &'static str {
+ if error.is_timeout() {
+ "timeout"
+ } else if error.is_decode() {
+ "request_error"
+ } else if error.is_connect() || error.is_request() || error.is_body() {
+ "transport"
+ } else {
+ "request_error"
+ }
+}
+
+/// Build one request body for a public Switchyard API.
+pub fn request_body(
+ endpoint: Endpoint,
+ model: &str,
+ prompt: &str,
+ max_output_tokens: u32,
+ stream: bool,
+) -> Value {
+ match endpoint {
+ // Chat Completions and Anthropic Messages take the same model/messages/max_tokens body.
+ Endpoint::Chat | Endpoint::Messages => json!({
+ "model": model,
+ "messages": [{"role": "user", "content": prompt}],
+ "max_tokens": max_output_tokens,
+ "temperature": 0,
+ "stream": stream,
+ }),
+ Endpoint::Responses => json!({
+ "model": model,
+ "input": prompt,
+ "max_output_tokens": max_output_tokens,
+ "stream": stream,
+ }),
+ }
+}
+
+#[derive(Default)]
+struct Metrics {
+ requests: Option<f64>,
+ errors: Option<f64>,
+}
+
+fn parse_metrics(text: &str) -> Metrics {
+ let mut metrics = Metrics::default();
+ for line in text.lines() {
+ if line.is_empty() || line.starts_with('#') {
+ continue;
+ }
+ let Some((name_part, rest)) = line.split_once(' ') else {
+ continue;
+ };
+ let name = name_part.split('{').next().unwrap_or(name_part);
+ if let Some(token) = rest.split_whitespace().next()
+ && let Ok(value) = token.parse::<f64>()
+ {
+ match name {
+ SERVER_REQUESTS_METRIC => metrics.requests = Some(value),
+ SERVER_ERRORS_METRIC => metrics.errors = Some(value),
+ _ => {}
+ }
+ }
+ }
+ metrics
+}
+
+#[derive(Debug)]
+pub struct RequestError {
+ pub kind: String,
+ pub detail: String,
+}
+
+impl RequestError {
+ fn new(kind: impl Into<String>, detail: impl Into<String>) -> Self {
+ Self {
+ kind: kind.into(),
+ detail: detail.into(),
+ }
+ }
+
+ fn transport(error: &reqwest::Error) -> Self {
+ Self::new(
+ transport_error_kind(error),
+ truncate(&error.to_string(), 500),
+ )
+ }
+}
+
+struct StreamValidator {
+ endpoint: Endpoint,
+ event_name: Option<String>,
+ saw_json: bool,
+ saw_terminal: bool,
+}
+
+impl StreamValidator {
+ fn new(endpoint: Endpoint) -> Self {
+ Self {
+ endpoint,
+ event_name: None,
+ saw_json: false,
+ saw_terminal: false,
+ }
+ }
+
+ fn read_line(&mut self, line: &[u8]) -> Result<(), RequestError> {
+ let line = line.strip_suffix(b"\r").unwrap_or(line);
+ if line.is_empty() {
+ self.event_name = None;
+ return Ok(());
+ }
+ let line = std::str::from_utf8(line)
+ .map_err(|error| RequestError::new("invalid_stream", error.to_string()))?;
+ if let Some(event_name) = line.strip_prefix("event:") {
+ self.event_name = Some(event_name.trim_start().to_string());
+ return Ok(());
+ }
+ let Some(data) = line.strip_prefix("data:") else {
+ return Ok(());
+ };
+ let data = data.strip_prefix(' ').unwrap_or(data);
+ if self.saw_terminal {
+ return Err(RequestError::new(
+ "invalid_stream",
+ "stream contained data after its terminal event",
+ ));
+ }
+ if data == "[DONE]" {
+ if self.endpoint != Endpoint::Chat {
+ return Err(RequestError::new(
+ "invalid_stream",
+ format!(
+ "{} stream contained an OpenAI [DONE] marker",
+ self.endpoint.as_str()
+ ),
+ ));
+ }
+ self.saw_terminal = true;
+ return Ok(());
+ }
+
+ let payload: Value = serde_json::from_str(data).map_err(|error| {
+ RequestError::new("invalid_stream", format!("invalid SSE JSON: {error}"))
+ })?;
+ let event_type = payload.get("type").and_then(Value::as_str);
+ if self.event_name.as_deref() == Some("error")
+ || event_type == Some("error")
+ || payload.get("error").is_some()
+ {
+ let detail = payload
+ .pointer("/error/message")
+ .or_else(|| payload.get("message"))
+ .and_then(Value::as_str)
+ .unwrap_or("stream returned an error event");
+ return Err(RequestError::new("stream_error", truncate(detail, 500)));
+ }
+
+ match self.endpoint {
+ Endpoint::Chat => {}
+ Endpoint::Messages | Endpoint::Responses => {
+ let event_type = event_type.ok_or_else(|| {
+ RequestError::new("invalid_stream", "SSE data did not contain an event type")
+ })?;
+ if self.event_name.as_deref() != Some(event_type) {
+ return Err(RequestError::new(
+ "invalid_stream",
+ format!(
+ "SSE event name {:?} did not match data type {event_type:?}",
+ self.event_name
+ ),
+ ));
+ }
+ self.saw_terminal = matches!(
+ (self.endpoint, event_type),
+ (Endpoint::Messages, "message_stop")
+ | (
+ Endpoint::Responses,
+ "response.completed" | "response.incomplete"
+ )
+ );
+ }
+ }
+ self.saw_json = true;
+ Ok(())
+ }
+
+ fn finish(self) -> Result<(), RequestError> {
+ if !self.saw_json {
+ return Err(RequestError::new(
+ "empty_stream",
+ "successful streaming response contained no JSON events",
+ ));
+ }
+ if !self.saw_terminal {
+ return Err(RequestError::new(
+ "invalid_stream",
+ format!(
+ "{} stream ended without its terminal event",
+ self.endpoint.as_str()
+ ),
+ ));
+ }
+ Ok(())
+ }
+}
+
+/// Send one request and validate its response body or event stream.
+pub async fn send_request(
+ client: &Client,
+ base_url: &str,
+ endpoint: Endpoint,
+ session_id: &str,
+ body: &Value,
+) -> Result<(), RequestError> {
+ let url = format!("{base_url}{}", endpoint.path());
+ let stream = body.get("stream").and_then(Value::as_bool).unwrap_or(false);
+ let response = match client
+ .post(&url)
+ .header("x-switchyard-session-id", session_id)
+ .json(body)
+ .send()
+ .await
+ {
+ Ok(response) => response,
+ Err(error) => return Err(RequestError::transport(&error)),
+ };
+
+ if stream {
+ let status = response.status();
+ if !status.is_success() {
+ let content = response.text().await.unwrap_or_default();
+ return Err(RequestError::new(
+ format!("http_{}", status.as_u16()),
+ truncate(&content, 500),
+ ));
+ }
+ let content_type = response
+ .headers()
+ .get(CONTENT_TYPE)
+ .and_then(|value| value.to_str().ok())
+ .unwrap_or("")
+ .to_string();
+ if !content_type.contains("text/event-stream") {
+ let content = response.text().await.unwrap_or_default();
+ return Err(RequestError::new(
+ "invalid_stream",
+ format!(
+ "expected text/event-stream, received {content_type:?}: {}",
+ truncate(&content, 300)
+ ),
+ ));
+ }
+ let mut bytes = response.bytes_stream();
+ let mut pending = Vec::new();
+ let mut validator = StreamValidator::new(endpoint);
+ while let Some(chunk) = bytes.next().await {
+ match chunk {
+ Ok(chunk) => {
+ pending.extend_from_slice(&chunk);
+ while let Some(newline) = pending.iter().position(|byte| *byte == b'\n') {
+ let mut line = pending.drain(..=newline).collect::<Vec<_>>();
+ line.pop();
+ validator.read_line(&line)?;
+ }
+ }
+ Err(error) => {
+ return Err(RequestError::transport(&error));
+ }
+ }
+ }
+ if !pending.is_empty() {
+ validator.read_line(&pending)?;
+ }
+ return validator.finish();
+ }
+
+ let status = response.status();
+ if !status.is_success() {
+ let content = response.text().await.unwrap_or_default();
+ return Err(RequestError::new(
+ format!("http_{}", status.as_u16()),
+ truncate(&content, 500),
+ ));
+ }
+ let text = match response.text().await {
+ Ok(text) => text,
+ Err(error) => return Err(RequestError::transport(&error)),
+ };
+ let payload: Value = match serde_json::from_str(&text) {
+ Ok(payload) => payload,
+ Err(error) => return Err(RequestError::new("invalid_json", error.to_string())),
+ };
+ if !payload.is_object() || payload.get("error").is_some() {
+ return Err(RequestError::new("invalid_response", truncate(&text, 500)));
+ }
+ let field = endpoint.required_field();
+ if payload.get(field).is_none() {
+ return Err(RequestError::new(
+ "invalid_response",
+ format!(
+ "successful {} response did not contain {field:?}: {}",
+ endpoint.as_str(),
+ truncate(&text, 300)
+ ),
+ ));
+ }
+ Ok(())
+}
+
+/// Check liveness and verify that the requested model is advertised.
+pub async fn preflight(client: &Client, base_url: &str, model: &str) -> Result<(), String> {
+ let health = client
+ .get(format!("{base_url}/health"))
+ .send()
+ .await
+ .map_err(|error| error.to_string())?
+ .error_for_status()
+ .map_err(|error| error.to_string())?;
+ let health_text = health.text().await.map_err(|error| error.to_string())?;
+ let health_body: Value = serde_json::from_str(&health_text).map_err(|_| {
+ format!(
+ "GET /health did not return JSON: {}",
+ truncate(&health_text, 300)
+ )
+ })?;
+ if health_body.get("status").and_then(Value::as_str) != Some("ok") {
+ return Err(format!(
+ "GET /health returned an unexpected body: {}",
+ truncate(&health_text, 300)
+ ));
+ }
+
+ let response = client
+ .get(format!("{base_url}/v1/models"))
+ .send()
+ .await
+ .map_err(|error| error.to_string())?
+ .error_for_status()
+ .map_err(|error| error.to_string())?;
+ let text = response.text().await.map_err(|error| error.to_string())?;
+ let body: Value = serde_json::from_str(&text).map_err(|_| {
+ format!(
+ "GET /v1/models did not return JSON: {}",
+ truncate(&text, 300)
+ )
+ })?;
+ let entries = body.get("data").and_then(Value::as_array).ok_or_else(|| {
+ format!(
+ "GET /v1/models returned an unexpected body: {}",
+ truncate(&text, 300)
+ )
+ })?;
+ if !entries
+ .iter()
+ .any(|entry| entry.get("id").and_then(Value::as_str) == Some(model))
+ {
+ return Err(format!("model {model:?} is not listed by GET /v1/models"));
+ }
+ Ok(())
+}
+
+pub struct ServerState {
+ pub healthy: bool,
+ pub requests: Option<f64>,
+ pub errors: Option<f64>,
+}
+
+/// Read liveness and cumulative metrics; one bad read becomes one failed sample.
+pub async fn read_server_state(client: &Client, base_url: &str) -> ServerState {
+ let healthy = match client.get(format!("{base_url}/health")).send().await {
+ Ok(response) if response.status() == reqwest::StatusCode::OK => response
+ .text()
+ .await
+ .ok()
+ .and_then(|text| serde_json::from_str::<Value>(&text).ok())
+ .and_then(|body| {
+ body.get("status")
+ .and_then(Value::as_str)
+ .map(|s| s == "ok")
+ })
+ .unwrap_or(false),
+ _ => false,
+ };
+ let metrics = match client.get(format!("{base_url}/metrics")).send().await {
+ Ok(response) if response.status().is_success() => response
+ .text()
+ .await
+ .map(|text| parse_metrics(&text))
+ .unwrap_or_default(),
+ _ => Metrics::default(),
+ };
+ ServerState {
+ healthy,
+ requests: metrics.requests,
+ errors: metrics.errors,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn request_bodies_match_public_endpoints() {
+ let chat = request_body(Endpoint::Chat, "route", "hello", 8, true);
+ let messages = request_body(Endpoint::Messages, "route", "hello", 8, true);
+ let responses = request_body(Endpoint::Responses, "route", "hello", 8, true);
+
+ assert_eq!(chat["messages"][0]["content"], json!("hello"));
+ assert_eq!(chat["max_tokens"], json!(8));
+ assert_eq!(messages["max_tokens"], json!(8));
+ assert_eq!(responses["input"], json!("hello"));
+ assert_eq!(responses["max_output_tokens"], json!(8));
+ for body in [&chat, &messages, &responses] {
+ assert_eq!(body["stream"], json!(true));
+ }
+ }
+
+ #[test]
+ fn parse_metrics_reads_only_required_counters() {
+ let metrics = parse_metrics(
+ "# TYPE switchyard_total_requests gauge\n\
+ switchyard_total_requests 42\n\
+ switchyard_total_errors{} 3\n\
+ switchyard_requests_total{model=\"route\"} 10\n",
+ );
+
+ assert_eq!(metrics.requests, Some(42.0));
+ assert_eq!(metrics.errors, Some(3.0));
+ }
+
+ #[test]
+ fn stream_validator_accepts_each_public_terminal_event() -> Result<(), RequestError> {
+ for (endpoint, stream) in [
+ (Endpoint::Chat, "data: {\"choices\":[]}\n\ndata: [DONE]\n\n"),
+ (
+ Endpoint::Messages,
+ "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n",
+ ),
+ (
+ Endpoint::Responses,
+ "event: response.completed\ndata: {\"type\":\"response.completed\"}\n\n",
+ ),
+ ] {
+ let mut validator = StreamValidator::new(endpoint);
+ for line in stream.split('\n') {
+ validator.read_line(line.as_bytes())?;
+ }
+ validator.finish()?;
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn stream_validator_rejects_errors_and_missing_terminal_events() {
+ let mut error = StreamValidator::new(Endpoint::Chat);
+ let result = error.read_line(b"data: {\"error\":{\"message\":\"boom\"}}");
+ assert_eq!(result.unwrap_err().kind, "stream_error");
+
+ let mut incomplete = StreamValidator::new(Endpoint::Responses);
+ incomplete
+ .read_line(b"event: response.output_text.delta")
+ .expect("event name should parse");
+ incomplete
+ .read_line(b"data: {\"type\":\"response.output_text.delta\",\"delta\":\"OK\"}")
+ .expect("data should parse");
+ assert_eq!(incomplete.finish().unwrap_err().kind, "invalid_stream");
+ }
+}
diff --git a/crates/switchyard-soak/src/lib.rs b/crates/switchyard-soak/src/lib.rs
new file mode 100644
index 000000000..779b70093
--- /dev/null
+++ b/crates/switchyard-soak/src/lib.rs
@@ -0,0 +1,565 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+#![doc = include_str!("../README.md")]
+
+mod client;
+mod report;
+mod scenarios;
+mod stats;
+
+use std::fs;
+use std::future::Future;
+use std::path::{Path, PathBuf};
+use std::process::ExitCode;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::{Duration, Instant};
+
+use clap::Parser;
+use parking_lot::Mutex;
+use reqwest::Client;
+use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue};
+use serde_json::json;
+use tokio::sync::Notify;
+
+use crate::client::{Endpoint, preflight, send_request};
+use crate::report::{ResultsWriter, invalid_request_canary, reporter};
+use crate::scenarios::{Scenario, ScenarioOptions, ScenarioSet};
+use crate::stats::{RunStats, build_summary, now_utc_string, round3, utc_dir_stamp};
+
+/// Command-line arguments for the soak test.
+#[derive(Parser)]
+#[command(
+ name = "switchyard-soak",
+ about = "Run a sustained, closed-loop load test against a live Switchyard server",
+ after_long_help = "Examples:\n switchyard-soak --model switchyard/general --duration 5m --concurrency 4\n switchyard-soak --model switchyard/general --scenario tool-call-burst --duration 5m\n switchyard-soak --model switchyard/general --export-scenarios scenarios\n\nThis command does not require oha or AIPerf. scripts/benchmark_routing_algorithms.py uses the exported scenario files with both tools.",
+ version
+)]
+pub struct Args {
+ /// HTTP base URL of the Switchyard server under test.
+ #[arg(long, default_value = "http://127.0.0.1:4000")]
+ base_url: String,
+
+ /// Exact route id advertised by GET /v1/models.
+ #[arg(long)]
+ model: String,
+
+ /// Time to keep sending load; use an s, m, or h suffix.
+ #[arg(long, value_parser = stats::parse_duration, default_value = "48h")]
+ duration: f64,
+
+ /// Requests kept in flight; each worker sends its next request after the last one ends.
+ #[arg(long, default_value_t = 16)]
+ concurrency: usize,
+
+ /// Default output-token limit; decode-heavy uses bounded 512 and 1,024 limits.
+ #[arg(long, default_value_t = 32)]
+ max_output_tokens: u32,
+
+ /// Bytes of repeated prefix added to each prompt to exercise request memory and caching.
+ #[arg(long, default_value_t = 1024)]
+ prompt_bytes: usize,
+
+ /// Named scenario collection; explicit --scenario values take precedence.
+ #[arg(long, value_enum, default_value_t = ScenarioSet::Standard)]
+ scenario_set: ScenarioSet,
+
+ /// Exact scenario id to run; repeat to select more than one scenario.
+ #[arg(long, action = clap::ArgAction::Append)]
+ scenario: Vec<String>,
+
+ /// Context-window size used to bound generated long-context scenarios.
+ #[arg(long, default_value_t = 32_768)]
+ context_window_tokens: usize,
+
+ /// Export selected AIPerf input files and exit without contacting a server.
+ #[arg(long)]
+ export_scenarios: Option<PathBuf>,
+
+ /// Seconds allowed to connect or wait between response bytes; an active stream may run longer.
+ #[arg(long, default_value_t = 120.0)]
+ request_timeout: f64,
+
+ /// Seconds between health, metrics, process, and result samples.
+ #[arg(long, default_value_t = 60.0)]
+ report_interval: f64,
+
+ /// Seconds between malformed-request checks; 0 turns this check off.
+ #[arg(long, default_value_t = 300.0)]
+ invalid_canary_interval: f64,
+
+ /// Largest passing request-error fraction, from 0 (none) through 1 (all).
+ #[arg(long, default_value_t = 0.0)]
+ max_error_rate: f64,
+
+ /// PID of a local switchyard-server process whose RSS and CPU should be sampled.
+ #[arg(long)]
+ server_pid: Option<u32>,
+
+ /// Largest passing first-to-last RSS increase in MiB; requires --server-pid.
+ #[arg(long, requires = "server_pid")]
+ max_rss_growth_mib: Option<f64>,
+
+ /// Name of an environment variable that holds the endpoint's bearer token.
+ #[arg(long)]
+ api_key_env: Option<String>,
+
+ /// New directory to create for config, interval, error, and summary files.
+ #[arg(long)]
+ results_dir: Option<PathBuf>,
+}
+
+impl Args {
+ /// Reject invalid numeric combinations after clap parses their types.
+ pub fn validate(&self) -> Result<(), String> {
+ if self.concurrency == 0 {
+ return Err("--concurrency must be greater than zero".to_string());
+ }
+ if self.max_output_tokens == 0 || self.prompt_bytes == 0 {
+ return Err(
+ "--max-output-tokens and --prompt-bytes must be greater than zero".to_string(),
+ );
+ }
+ ScenarioOptions {
+ model: &self.model,
+ prompt_bytes: self.prompt_bytes,
+ max_output_tokens: self.max_output_tokens,
+ context_window_tokens: self.context_window_tokens,
+ }
+ .validate()?;
+ if !self.request_timeout.is_finite()
+ || !self.report_interval.is_finite()
+ || self.request_timeout <= 0.0
+ || self.report_interval <= 0.0
+ {
+ return Err(
+ "--request-timeout and --report-interval must be greater than zero".to_string(),
+ );
+ }
+ if !self.invalid_canary_interval.is_finite() || self.invalid_canary_interval < 0.0 {
+ return Err("--invalid-canary-interval must be zero or greater".to_string());
+ }
+ if !(0.0..=1.0).contains(&self.max_error_rate) {
+ return Err("--max-error-rate must be between 0 and 1".to_string());
+ }
+ if self.server_pid == Some(0) {
+ return Err("--server-pid must be greater than zero".to_string());
+ }
+ if self
+ .max_rss_growth_mib
+ .is_some_and(|growth| !growth.is_finite() || growth < 0.0)
+ {
+ return Err("--max-rss-growth-mib must be zero or greater".to_string());
+ }
+ Ok(())
+ }
+}
+
+#[derive(Clone)]
+struct RunContext {
+ client: Client,
+ base_url: String,
+ stop: Arc<Stop>,
+ stats: Arc<Mutex<RunStats>>,
+ writer: Arc<Mutex<ResultsWriter>>,
+}
+
+struct Workload {
+ model: String,
+ scenarios: Vec<Scenario>,
+}
+
+/// A one-shot stop signal that many tasks can wait on and any task can raise.
+struct Stop {
+ flag: AtomicBool,
+ notify: Notify,
+}
+
+impl Stop {
+ fn new() -> Self {
+ Self {
+ flag: AtomicBool::new(false),
+ notify: Notify::new(),
+ }
+ }
+
+ fn set(&self) {
+ self.flag.store(true, Ordering::SeqCst);
+ self.notify.notify_waiters();
+ }
+
+ fn is_set(&self) -> bool {
+ self.flag.load(Ordering::SeqCst)
+ }
+
+ /// Resolve once the signal is raised, now or later.
+ async fn wait(&self) {
+ loop {
+ if self.is_set() {
+ return;
+ }
+ // Register before the second flag check so a set() between the two still wakes us.
+ let notified = self.notify.notified();
+ tokio::pin!(notified);
+ notified.as_mut().enable();
+ if self.is_set() {
+ return;
+ }
+ notified.await;
+ }
+ }
+}
+
+/// Send closed-loop traffic until the run stops.
+async fn worker(
+ context: RunContext,
+ workload: Arc<Workload>,
+ worker_id: usize,
+) -> Result<(), String> {
+ let mut request_number = 0;
+ while !context.stop.is_set() {
+ let scenario = &workload.scenarios[request_number % workload.scenarios.len()];
+ let stream = (request_number / workload.scenarios.len()).is_multiple_of(2);
+ let request = scenario.request(request_number / workload.scenarios.len(), stream);
+ let session_id = format!("{}-worker-{worker_id}", request.session_id);
+ let started = Instant::now();
+ let result = send_request(
+ &context.client,
+ &context.base_url,
+ request.endpoint,
+ &session_id,
+ request.body,
+ )
+ .await;
+ let latency_ms = started.elapsed().as_secs_f64() * 1000.0;
+ context.stats.lock().record(
+ request.endpoint.as_str(),
+ scenario.id,
+ latency_ms,
+ result.as_ref().err().map(|error| error.kind.as_str()),
+ );
+ if let Err(error) = result {
+ context
+ .writer
+ .lock()
+ .write_error(&json!({
+ "timestamp_utc": now_utc_string(),
+ "worker": worker_id,
+ "scenario": scenario.id,
+ "endpoint": request.endpoint.as_str(),
+ "stream": stream,
+ "latency_ms": round3(latency_ms),
+ "error": error.kind,
+ "detail": error.detail,
+ }))
+ .map_err(|error| error.to_string())?;
+ }
+ request_number = request_number.wrapping_add(1);
+ }
+ Ok(())
+}
+
+/// Stop the run when a background task fails or panics.
+async fn guard_stop(
+ stop: Arc<Stop>,
+ task: impl Future<Output = Result<(), String>>,
+) -> Result<(), String> {
+ struct StopOnDrop(Arc<Stop>, bool);
+ impl Drop for StopOnDrop {
+ fn drop(&mut self) {
+ if self.1 {
+ self.0.set();
+ }
+ }
+ }
+ let mut guard = StopOnDrop(stop, true);
+ let result = task.await;
+ guard.1 = result.is_err();
+ result
+}
+
+/// Add one joined task's error to the failure reasons; cancelled tasks are expected.
+fn collect_failure(
+ name: &str,
+ result: Result<Result<(), String>, tokio::task::JoinError>,
+ failures: &mut Vec<String>,
+) {
+ match result {
+ Ok(Ok(())) => {}
+ Ok(Err(reason)) => failures.push(format!("{name} failed: {reason}")),
+ Err(join) if join.is_cancelled() => {}
+ Err(join) => failures.push(format!("{name} failed: {join}")),
+ }
+}
+
+/// Raise *stop* on SIGINT or SIGTERM so an operator can end the run cleanly.
+fn spawn_signal_listener(stop: Arc<Stop>) -> tokio::task::JoinHandle<()> {
+ tokio::spawn(async move {
+ #[cfg(unix)]
+ {
+ use tokio::signal::unix::{SignalKind, signal};
+ let mut interrupt = signal(SignalKind::interrupt()).ok();
+ let mut terminate = signal(SignalKind::terminate()).ok();
+ let wait_interrupt = async {
+ match interrupt.as_mut() {
+ Some(stream) => {
+ stream.recv().await;
+ }
+ None => std::future::pending::<()>().await,
+ }
+ };
+ let wait_terminate = async {
+ match terminate.as_mut() {
+ Some(stream) => {
+ stream.recv().await;
+ }
+ None => std::future::pending::<()>().await,
+ }
+ };
+ tokio::select! {
+ _ = wait_interrupt => {}
+ _ = wait_terminate => {}
+ }
+ stop.set();
+ }
+ #[cfg(not(unix))]
+ {
+ let _ = tokio::signal::ctrl_c().await;
+ stop.set();
+ }
+ })
+}
+
+/// Record every non-secret input with the normalized duration and fixed request variants.
+fn write_config(
+ results_dir: &Path,
+ args: &Args,
+ model: &str,
+ scenarios: &[Scenario],
+) -> Result<(), String> {
+ let config = json!({
+ "base_url": args.base_url,
+ "model": model,
+ "duration_seconds": args.duration,
+ "concurrency": args.concurrency,
+ "endpoints": Endpoint::ALL.map(Endpoint::as_str),
+ "streaming": [true, false],
+ "max_output_tokens": args.max_output_tokens,
+ "prompt_bytes": args.prompt_bytes,
+ "context_window_tokens": args.context_window_tokens,
+ "scenarios": scenarios.iter().map(|scenario| scenario.id).collect::<Vec<_>>(),
+ "request_timeout": args.request_timeout,
+ "report_interval": args.report_interval,
+ "invalid_canary_interval": args.invalid_canary_interval,
+ "max_error_rate": args.max_error_rate,
+ "server_pid": args.server_pid,
+ "max_rss_growth_mib": args.max_rss_growth_mib,
+ "api_key_env": args.api_key_env,
+ });
+ let body = serde_json::to_string_pretty(&config).map_err(|error| error.to_string())?;
+ fs::write(results_dir.join("config.json"), format!("{body}\n"))
+ .map_err(|error| error.to_string())
+}
+
+/// Run the configured soak test and return a process exit code (0 pass, 1 fail).
+pub async fn run(args: Args) -> Result<i32, String> {
+ args.validate()?;
+
+ let scenarios = scenarios::select(
+ ScenarioOptions {
+ model: &args.model,
+ prompt_bytes: args.prompt_bytes,
+ max_output_tokens: args.max_output_tokens,
+ context_window_tokens: args.context_window_tokens,
+ },
+ args.scenario_set,
+ &args.scenario,
+ )?;
+ if let Some(output_dir) = &args.export_scenarios {
+ let manifest = scenarios::export(&scenarios, &args.model, output_dir)?;
+ println!("Exported scenario manifest: {}", manifest.display());
+ return Ok(0);
+ }
+
+ let token = match &args.api_key_env {
+ Some(var) => {
+ let token = std::env::var(var).ok().filter(|value| !value.is_empty());
+ if token.is_none() {
+ return Err(format!("${var} is not set"));
+ }
+ token
+ }
+ None => None,
+ };
+
+ // Per-operation timeouts, not a whole-request deadline: a healthy long stream that keeps
+ // delivering bytes must not be aborted, so bound connect time and idle time between reads.
+ let request_timeout = Duration::from_secs_f64(args.request_timeout);
+ let mut builder = Client::builder()
+ .no_proxy()
+ .connect_timeout(request_timeout)
+ .read_timeout(request_timeout)
+ .pool_max_idle_per_host(args.concurrency + 4);
+ if let Some(token) = &token {
+ let mut headers = HeaderMap::new();
+ let value = HeaderValue::from_str(&format!("Bearer {token}"))
+ .map_err(|_| "the bearer token is not a valid HTTP header value".to_string())?;
+ headers.insert(AUTHORIZATION, value);
+ builder = builder.default_headers(headers);
+ }
+ let client = builder.build().map_err(|error| error.to_string())?;
+ let base_url = args.base_url.trim_end_matches('/').to_string();
+
+ preflight(&client, &base_url, &args.model).await?;
+ let results_dir = args
+ .results_dir
+ .clone()
+ .unwrap_or_else(|| PathBuf::from("soak-results").join(utc_dir_stamp()));
+ let writer = Arc::new(Mutex::new(
+ ResultsWriter::new(&results_dir).map_err(|error| error.to_string())?,
+ ));
+ write_config(&results_dir, &args, &args.model, &scenarios)?;
+
+ println!(
+ "Soak started: model={} duration={}s concurrency={} scenarios={} results={}",
+ args.model,
+ args.duration,
+ args.concurrency,
+ scenarios
+ .iter()
+ .map(|scenario| scenario.id)
+ .collect::<Vec<_>>()
+ .join(","),
+ results_dir.display(),
+ );
+
+ let started = Instant::now();
+ let stats = Arc::new(Mutex::new(RunStats::new(2026)));
+ let stop = Arc::new(Stop::new());
+ let workers_done = Arc::new(Stop::new());
+ let context = RunContext {
+ client,
+ base_url,
+ stop: stop.clone(),
+ stats: stats.clone(),
+ writer: writer.clone(),
+ };
+
+ let signal_listener = spawn_signal_listener(stop.clone());
+
+ let deadline = {
+ let stop = stop.clone();
+ let stats = stats.clone();
+ let duration = Duration::from_secs_f64(args.duration);
+ tokio::spawn(async move {
+ tokio::time::sleep(duration).await;
+ stats.lock().completed_duration = true;
+ stop.set();
+ })
+ };
+
+ let workload = Arc::new(Workload {
+ model: args.model.clone(),
+ scenarios,
+ });
+ let mut worker_handles = Vec::new();
+ for worker_id in 0..args.concurrency {
+ let handle = tokio::spawn(guard_stop(
+ stop.clone(),
+ worker(context.clone(), workload.clone(), worker_id),
+ ));
+ worker_handles.push((format!("worker-{worker_id}"), handle));
+ }
+ let reporter_handle = tokio::spawn(guard_stop(
+ stop.clone(),
+ reporter(
+ context.clone(),
+ started,
+ Duration::from_secs_f64(args.report_interval),
+ args.duration,
+ args.server_pid,
+ workers_done.clone(),
+ ),
+ ));
+ let canary_handle = tokio::spawn(guard_stop(
+ stop.clone(),
+ invalid_request_canary(context, args.invalid_canary_interval, workload),
+ ));
+
+ stop.wait().await;
+ deadline.abort();
+ signal_listener.abort();
+
+ // A crashed worker/reporter/canary must not discard the run: record it as a failure reason
+ // so the summary is still written and the run fails closed.
+ let mut task_failures = Vec::new();
+ for (name, handle) in worker_handles {
+ collect_failure(&name, handle.await, &mut task_failures);
+ }
+ workers_done.set();
+ collect_failure("reporter", reporter_handle.await, &mut task_failures);
+ collect_failure(
+ "invalid-request-canary",
+ canary_handle.await,
+ &mut task_failures,
+ );
+
+ let elapsed = started.elapsed().as_secs_f64();
+ let (error_records, dropped_error_records) = { writer.lock().error_counts() };
+ let summary = build_summary(
+ &stats.lock(),
+ elapsed,
+ args.max_error_rate,
+ args.max_rss_growth_mib,
+ error_records,
+ dropped_error_records,
+ &task_failures,
+ );
+ let summary_path = results_dir.join("summary.json");
+ let summary_body = serde_json::to_string_pretty(&summary).map_err(|error| error.to_string())?;
+ fs::write(&summary_path, format!("{summary_body}\n")).map_err(|error| error.to_string())?;
+
+ let label = if summary.passed { "PASS" } else { "FAIL" };
+ println!(
+ "Soak {label}: requests={} error_rate={:.4}% p95_ms={} summary={}",
+ summary.requests,
+ summary.error_rate * 100.0,
+ summary
+ .latency_p95_ms
+ .map(|value| value.to_string())
+ .unwrap_or_default(),
+ summary_path.display(),
+ );
+ for reason in &summary.failure_reasons {
+ println!("- {reason}");
+ }
+ Ok(if summary.passed { 0 } else { 1 })
+}
+
+/// Parse arguments, run the test on a multi-thread runtime, and map the result to an exit code.
+pub fn cli_main() -> ExitCode {
+ let args = Args::parse();
+ if let Err(message) = args.validate() {
+ eprintln!("switchyard-soak: {message}");
+ return ExitCode::from(2);
+ }
+ let runtime = match tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()
+ {
+ Ok(runtime) => runtime,
+ Err(error) => {
+ eprintln!("soak test setup failed: {error}");
+ return ExitCode::from(2);
+ }
+ };
+ match runtime.block_on(run(args)) {
+ Ok(0) => ExitCode::SUCCESS,
+ Ok(_) => ExitCode::from(1),
+ Err(error) => {
+ eprintln!("soak test setup failed: {error}");
+ ExitCode::from(2)
+ }
+ }
+}
diff --git a/crates/switchyard-soak/src/main.rs b/crates/switchyard-soak/src/main.rs
new file mode 100644
index 000000000..8e3d676d5
--- /dev/null
+++ b/crates/switchyard-soak/src/main.rs
@@ -0,0 +1,10 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Binary entrypoint for `switchyard-soak`.
+
+use std::process::ExitCode;
+
+fn main() -> ExitCode {
+ switchyard_soak::cli_main()
+}
diff --git a/crates/switchyard-soak/src/report.rs b/crates/switchyard-soak/src/report.rs
new file mode 100644
index 000000000..69e512528
--- /dev/null
+++ b/crates/switchyard-soak/src/report.rs
@@ -0,0 +1,327 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Result files plus the background reporter and invalid-request canary tasks.
+
+use std::fs::{self, File};
+use std::io::{self, Write};
+use std::path::Path;
+use std::time::{Duration, Instant};
+
+use serde_json::{Value, json};
+
+use crate::client::read_server_state;
+use crate::stats::{latency_stats, now_utc_string, round3};
+use crate::{RunContext, Stop, Workload};
+
+/// Cap on individually recorded failures so a bad run cannot fill the disk.
+const MAX_ERROR_RECORDS: u64 = 10_000;
+
+const INTERVAL_FIELDS: [&str; 15] = [
+ "timestamp_utc",
+ "elapsed_seconds",
+ "requests",
+ "successes",
+ "failures",
+ "requests_per_second",
+ "latency_p50_ms",
+ "latency_p95_ms",
+ "latency_p99_ms",
+ "latency_max_ms",
+ "health",
+ "server_total_requests",
+ "server_total_errors",
+ "rss_mib",
+ "cpu_percent",
+];
+
+/// Write interval rows and bounded error details as the run proceeds.
+pub struct ResultsWriter {
+ interval_file: File,
+ error_file: File,
+ error_records: u64,
+ dropped_error_records: u64,
+}
+
+impl ResultsWriter {
+ /// Create a fresh results directory; fails if it already exists.
+ pub fn new(results_dir: &Path) -> io::Result<Self> {
+ if let Some(parent) = results_dir.parent()
+ && !parent.as_os_str().is_empty()
+ {
+ fs::create_dir_all(parent)?;
+ }
+ fs::create_dir(results_dir)?;
+ let mut interval_file = File::create(results_dir.join("intervals.csv"))?;
+ writeln!(interval_file, "{}", INTERVAL_FIELDS.join(","))?;
+ let error_file = File::create(results_dir.join("errors.jsonl"))?;
+ Ok(Self {
+ interval_file,
+ error_file,
+ error_records: 0,
+ dropped_error_records: 0,
+ })
+ }
+
+ /// Append one interval row; `cells` are already formatted in `INTERVAL_FIELDS` order.
+ pub fn write_interval(&mut self, cells: &[String]) -> io::Result<()> {
+ writeln!(self.interval_file, "{}", cells.join(","))?;
+ self.interval_file.flush()
+ }
+
+ /// Append one failure record, dropping past the bound instead of growing without limit.
+ pub fn write_error(&mut self, record: &Value) -> io::Result<()> {
+ if self.error_records >= MAX_ERROR_RECORDS {
+ self.dropped_error_records += 1;
+ return Ok(());
+ }
+ writeln!(self.error_file, "{record}")?;
+ self.error_file.flush()?;
+ self.error_records += 1;
+ Ok(())
+ }
+
+ pub fn error_counts(&self) -> (u64, u64) {
+ (self.error_records, self.dropped_error_records)
+ }
+}
+
+/// Format an optional number for a CSV cell; `None` becomes an empty cell.
+fn cell(value: Option<f64>) -> String {
+ value
+ .map(|value| round3(value).to_string())
+ .unwrap_or_default()
+}
+
+/// Return RSS MiB and CPU percent for *pid* using the local `ps` command.
+pub async fn process_sample(pid: Option<u32>) -> (Option<f64>, Option<f64>) {
+ let Some(pid) = pid else {
+ return (None, None);
+ };
+ let output = match tokio::process::Command::new("ps")
+ .args(["-o", "rss=,pcpu=", "-p", &pid.to_string()])
+ .output()
+ .await
+ {
+ Ok(output) => output,
+ // ps missing or fork failed: record a process-check miss, don't crash the reporter.
+ Err(_) => return (None, None),
+ };
+ if !output.status.success() {
+ return (None, None);
+ }
+ let stdout = String::from_utf8_lossy(&output.stdout);
+ let mut fields = stdout.split_whitespace();
+ let (Some(rss), Some(cpu), None) = (fields.next(), fields.next(), fields.next()) else {
+ return (None, None);
+ };
+ match (rss.parse::<f64>(), cpu.parse::<f64>()) {
+ (Ok(rss_kib), Ok(cpu_percent)) => (Some(rss_kib / 1024.0), Some(cpu_percent)),
+ _ => (None, None),
+ }
+}
+
+/// Write one liveness, metrics, resource, and latency row per interval.
+pub async fn reporter(
+ context: RunContext,
+ started: Instant,
+ interval: Duration,
+ target_seconds: f64,
+ server_pid: Option<u32>,
+ workers_done: std::sync::Arc<Stop>,
+) -> Result<(), String> {
+ let mut previous_report = started;
+ loop {
+ tokio::select! {
+ _ = tokio::time::sleep(interval) => {}
+ _ = context.stop.wait() => {}
+ }
+ // Once stopping, wait for the workers to drain so the final row counts their last requests.
+ if context.stop.is_set() {
+ workers_done.wait().await;
+ }
+
+ let now = Instant::now();
+ let mut interval_stats = context.stats.lock().take_interval();
+ let server = read_server_state(&context.client, &context.base_url).await;
+ let (rss_mib, cpu_percent) = process_sample(server_pid).await;
+
+ let (total_successes, total_failures) = {
+ let mut state = context.stats.lock();
+ state.health_checks += 1;
+ state.metrics_checks += 1;
+ if !server.healthy {
+ state.health_failures += 1;
+ }
+ if server.requests.is_none() || server.errors.is_none() {
+ state.metrics_failures += 1;
+ }
+ if server_pid.is_some() {
+ state.process_checks += 1;
+ if rss_mib.is_none() || cpu_percent.is_none() {
+ state.process_failures += 1;
+ }
+ }
+ if let Some(rss) = rss_mib {
+ state.rss_samples.push(rss);
+ }
+ if let (Some(current), Some(previous)) =
+ (server.requests, state.previous_server_requests)
+ && current < previous
+ {
+ state.server_restarts += 1;
+ }
+ if let Some(current) = server.requests {
+ state.previous_server_requests = Some(current);
+ }
+ (state.total_successes, state.total_failures)
+ };
+
+ let elapsed_interval = (now - previous_report).as_secs_f64().max(0.001);
+ let requests = interval_stats.successes + interval_stats.failures;
+ let latency = latency_stats(&mut interval_stats.latencies_ms);
+ let requests_per_second = round3(requests as f64 / elapsed_interval);
+ let timestamp = now_utc_string();
+ let elapsed_seconds = round3((now - started).as_secs_f64());
+ let health_label = if server.healthy { "ok" } else { "failed" };
+
+ let cells = vec![
+ timestamp.clone(),
+ elapsed_seconds.to_string(),
+ requests.to_string(),
+ interval_stats.successes.to_string(),
+ interval_stats.failures.to_string(),
+ requests_per_second.to_string(),
+ cell(latency.p50_ms),
+ cell(latency.p95_ms),
+ cell(latency.p99_ms),
+ cell(latency.max_ms),
+ health_label.to_string(),
+ cell(server.requests),
+ cell(server.errors),
+ cell(rss_mib),
+ cell(cpu_percent),
+ ];
+ // Cumulative, progress, and a glanceable status token so a remote tail of the log shows
+ // at once that the run is alive, how far along it is, and whether it is healthy.
+ let cumulative_requests = total_successes + total_failures;
+ let cumulative_error_rate = if cumulative_requests > 0 {
+ total_failures as f64 / cumulative_requests as f64
+ } else {
+ 0.0
+ };
+ let progress = (elapsed_seconds / target_seconds).min(1.0);
+ let status = if requests == 0 {
+ "stalled"
+ } else if !server.healthy || interval_stats.failures > 0 {
+ "degraded"
+ } else {
+ "ok"
+ };
+ let p95_text = latency.p95_ms.map(|v| v.to_string()).unwrap_or_default();
+ let rss_text = rss_mib.map(|v| round3(v).to_string()).unwrap_or_default();
+
+ context
+ .writer
+ .lock()
+ .write_interval(&cells)
+ .map_err(|error| error.to_string())?;
+
+ println!(
+ "[{timestamp}] progress={elapsed_seconds:.0}s/{target_seconds:.0}s({:.0}%) \
+ reqs={cumulative_requests} interval={requests} errors={total_failures}({:.4}%) \
+ rps={requests_per_second} p95_ms={p95_text} health={health_label} rss_mib={rss_text} \
+ status={}",
+ progress * 100.0,
+ cumulative_error_rate * 100.0,
+ status.to_uppercase(),
+ );
+
+ previous_report = now;
+ if context.stop.is_set() {
+ return Ok(());
+ }
+ }
+}
+
+/// Confirm invalid input returns 400 and the server stays live, on a fixed interval.
+pub async fn invalid_request_canary(
+ context: RunContext,
+ interval: f64,
+ workload: std::sync::Arc<Workload>,
+) -> Result<(), String> {
+ if interval <= 0.0 {
+ return Ok(());
+ }
+ let interval = Duration::from_secs_f64(interval);
+ loop {
+ tokio::select! {
+ _ = tokio::time::sleep(interval) => {}
+ _ = context.stop.wait() => {}
+ }
+ if context.stop.is_set() {
+ return Ok(());
+ }
+
+ context.stats.lock().canaries += 1;
+ let probe = async {
+ let invalid = context
+ .client
+ .post(format!("{}/v1/chat/completions", context.base_url))
+ .json(&json!({"model": workload.model, "messages": "invalid"}))
+ .send()
+ .await?;
+ let health = context
+ .client
+ .get(format!("{}/health", context.base_url))
+ .send()
+ .await?;
+ Ok::<(u16, u16), reqwest::Error>((invalid.status().as_u16(), health.status().as_u16()))
+ }
+ .await;
+ let (passed, detail) = match probe {
+ Ok((invalid_status, health_status)) => (
+ invalid_status == 400 && health_status == 200,
+ format!("invalid_status={invalid_status}, health_status={health_status}"),
+ ),
+ Err(error) => (false, error.to_string()),
+ };
+ if !passed {
+ context.stats.lock().canary_failures += 1;
+ context
+ .writer
+ .lock()
+ .write_error(&json!({
+ "timestamp_utc": now_utc_string(),
+ "error": "invalid_request_canary",
+ "detail": detail,
+ }))
+ .map_err(|error| error.to_string())?;
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn error_details_stop_growing_at_the_file_limit() -> io::Result<()> {
+ let parent = tempfile::tempdir()?;
+ let results_dir = parent.path().join("results");
+ let mut writer = ResultsWriter::new(&results_dir)?;
+
+ for _ in 0..=MAX_ERROR_RECORDS {
+ writer.write_error(&serde_json::json!({"error": "upstream"}))?;
+ }
+
+ assert_eq!(writer.error_counts(), (MAX_ERROR_RECORDS, 1));
+ assert_eq!(
+ fs::read_to_string(results_dir.join("errors.jsonl"))?
+ .lines()
+ .count(),
+ 10_000
+ );
+ Ok(())
+ }
+}
diff --git a/crates/switchyard-soak/src/scenarios/classifier_mix.rs b/crates/switchyard-soak/src/scenarios/classifier_mix.rs
new file mode 100644
index 000000000..893366fdf
--- /dev/null
+++ b/crates/switchyard-soak/src/scenarios/classifier_mix.rs
@@ -0,0 +1,42 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::{
+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user,
+};
+
+pub fn build(options: ScenarioOptions<'_>) -> Scenario {
+ let sessions = (0..20)
+ .map(|index| {
+ let hard = (8..10).contains(&index) || index >= 15;
+ let marker = if hard {
+ "classifier_hard"
+ } else {
+ "classifier_easy"
+ };
+ Session {
+ session_id: format!("classifier-mix-{index}"),
+ payloads: vec![chat_payload(
+ options.model,
+ vec![user(format!(
+ "[scenario:{marker}] Request {index}: {}",
+ if hard {
+ "reason about a distributed failure with incomplete evidence"
+ } else {
+ "return the sum of two and two"
+ }
+ ))],
+ options.max_output_tokens,
+ )],
+ }
+ })
+ .collect();
+ Scenario::chat(
+ "classifier-mix",
+ ScenarioGroup::Agentic,
+ "Two deterministic easy/hard mixes: 80/20 followed by 50/50.",
+ "Selection shares match the input mix and classifier overhead is visible.",
+ ErrorExpectation::SUCCESS,
+ sessions,
+ )
+}
diff --git a/crates/switchyard-soak/src/scenarios/client_cancellation.rs b/crates/switchyard-soak/src/scenarios/client_cancellation.rs
new file mode 100644
index 000000000..66ccf29e3
--- /dev/null
+++ b/crates/switchyard-soak/src/scenarios/client_cancellation.rs
@@ -0,0 +1,26 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::{
+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user,
+};
+
+pub fn build(options: ScenarioOptions<'_>) -> Scenario {
+ Scenario::chat(
+ "client-cancellation",
+ ScenarioGroup::Resilience,
+ "Slow streaming requests whose client timeout cancels work in flight.",
+ "Cancellation releases connections and does not destabilize later traffic.",
+ ErrorExpectation::ALL,
+ vec![Session {
+ session_id: "client-cancellation".to_string(),
+ payloads: vec![chat_payload(
+ options.model,
+ vec![user(
+ "[scenario:client_cancellation] Delay this response until the client leaves.",
+ )],
+ options.max_output_tokens,
+ )],
+ }],
+ )
+}
diff --git a/crates/switchyard-soak/src/scenarios/context_overflow.rs b/crates/switchyard-soak/src/scenarios/context_overflow.rs
new file mode 100644
index 000000000..3e57e2007
--- /dev/null
+++ b/crates/switchyard-soak/src/scenarios/context_overflow.rs
@@ -0,0 +1,26 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::{
+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, token_text,
+ user,
+};
+
+pub fn build(options: ScenarioOptions<'_>) -> Scenario {
+ let tokens = (options.context_window_tokens * 9 / 10).max(1);
+ Scenario::chat(
+ "context-overflow",
+ ScenarioGroup::Resilience,
+ "A near-window request that one configured target rejects as too long.",
+ "A multi-target route retries an eligible target and returns a valid response.",
+ ErrorExpectation::SUCCESS,
+ vec![Session {
+ session_id: "context-overflow".to_string(),
+ payloads: vec![chat_payload(
+ options.model,
+ vec![user(token_text(tokens, "context_overflow"))],
+ options.max_output_tokens,
+ )],
+ }],
+ )
+}
diff --git a/crates/switchyard-soak/src/scenarios/decode_heavy.rs b/crates/switchyard-soak/src/scenarios/decode_heavy.rs
new file mode 100644
index 000000000..b19c4d68a
--- /dev/null
+++ b/crates/switchyard-soak/src/scenarios/decode_heavy.rs
@@ -0,0 +1,30 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::{
+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user,
+};
+
+pub fn build(options: ScenarioOptions<'_>) -> Scenario {
+ let sessions = [512, 1024]
+ .into_iter()
+ .map(|tokens| Session {
+ session_id: format!("decode-heavy-{tokens}"),
+ payloads: vec![chat_payload(
+ options.model,
+ vec![user(format!(
+ "[scenario:decode_heavy] Write {tokens} numbered one-word items."
+ ))],
+ tokens,
+ )],
+ })
+ .collect();
+ Scenario::chat(
+ "decode-heavy",
+ ScenarioGroup::Core,
+ "Short inputs with 512-token and 1,024-token output limits.",
+ "Output token throughput stays stable while long streams remain valid.",
+ ErrorExpectation::SUCCESS,
+ sessions,
+ )
+}
diff --git a/crates/switchyard-soak/src/scenarios/failure_pressure.rs b/crates/switchyard-soak/src/scenarios/failure_pressure.rs
new file mode 100644
index 000000000..a1606b065
--- /dev/null
+++ b/crates/switchyard-soak/src/scenarios/failure_pressure.rs
@@ -0,0 +1,36 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::{
+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user,
+};
+
+pub fn build(options: ScenarioOptions<'_>) -> Scenario {
+ let cases = [
+ ("truncated-stream", "truncated_stream"),
+ ("upstream-429", "upstream_429"),
+ ("upstream-500", "upstream_500"),
+ ("classifier-invalid", "classifier_invalid"),
+ ];
+ let sessions = cases
+ .into_iter()
+ .map(|(id, marker)| Session {
+ session_id: id.to_string(),
+ payloads: vec![chat_payload(
+ options.model,
+ vec![user(format!(
+ "[scenario:{marker}] Exercise the configured recovery path."
+ ))],
+ options.max_output_tokens,
+ )],
+ })
+ .collect();
+ Scenario::chat(
+ "failure-pressure",
+ ScenarioGroup::Resilience,
+ "Bounded 429, 500, malformed-classifier, and truncated-stream injections.",
+ "Retries recover transient failures; terminal failures remain explicit and bounded.",
+ ErrorExpectation::MIXED,
+ sessions,
+ )
+}
diff --git a/crates/switchyard-soak/src/scenarios/growing_conversation.rs b/crates/switchyard-soak/src/scenarios/growing_conversation.rs
new file mode 100644
index 000000000..17da931aa
--- /dev/null
+++ b/crates/switchyard-soak/src/scenarios/growing_conversation.rs
@@ -0,0 +1,34 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::{
+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, assistant, chat_payload,
+ user,
+};
+
+pub fn build(options: ScenarioOptions<'_>) -> Scenario {
+ let mut messages = Vec::new();
+ let mut payloads = Vec::new();
+ for turn in 0..8 {
+ messages.push(user(format!(
+ "[scenario:growing_conversation] Turn {turn}: remember the number {turn}."
+ )));
+ payloads.push(chat_payload(
+ options.model,
+ messages.clone(),
+ options.max_output_tokens.min(64),
+ ));
+ messages.push(assistant(format!("I recorded {turn}.")));
+ }
+ Scenario::chat(
+ "growing-conversation",
+ ScenarioGroup::Agentic,
+ "An eight-turn session whose complete message history grows each turn.",
+ "Session affinity holds and latency tracks the growing history.",
+ ErrorExpectation::SUCCESS,
+ vec![Session {
+ session_id: "growing-conversation".to_string(),
+ payloads,
+ }],
+ )
+}
diff --git a/crates/switchyard-soak/src/scenarios/large_tool_catalog.rs b/crates/switchyard-soak/src/scenarios/large_tool_catalog.rs
new file mode 100644
index 000000000..a95363f18
--- /dev/null
+++ b/crates/switchyard-soak/src/scenarios/large_tool_catalog.rs
@@ -0,0 +1,61 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use serde_json::{Value, json};
+
+use super::{
+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user,
+};
+
+fn tools(count: usize) -> Vec<Value> {
+ (0..count)
+ .map(|index| {
+ json!({
+ "type": "function",
+ "function": {
+ "name": format!("lookup_record_{index}"),
+ "description": format!("Look up record {index} in the test catalog."),
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "query": {"type": "string"},
+ "limit": {"type": "integer", "minimum": 1, "maximum": 10}
+ },
+ "required": ["query"],
+ "additionalProperties": false
+ }
+ }
+ })
+ })
+ .collect()
+}
+
+pub fn build(options: ScenarioOptions<'_>) -> Scenario {
+ let sessions = [16, 64]
+ .into_iter()
+ .map(|count| {
+ let mut payload = chat_payload(
+ options.model,
+ vec![user(format!(
+ "[scenario:large_tool_catalog] Use lookup_record_{} for query green.",
+ count - 1
+ ))],
+ options.max_output_tokens.min(128),
+ );
+ payload["tools"] = Value::Array(tools(count));
+ payload["tool_choice"] = json!("auto");
+ Session {
+ session_id: format!("large-tool-catalog-{count}"),
+ payloads: vec![payload],
+ }
+ })
+ .collect();
+ Scenario::chat(
+ "large-tool-catalog",
+ ScenarioGroup::Agentic,
+ "Requests with 16-tool and 64-tool JSON-schema catalogs.",
+ "Routing overhead remains bounded while large tool schemas are forwarded intact.",
+ ErrorExpectation::SUCCESS,
+ sessions,
+ )
+}
diff --git a/crates/switchyard-soak/src/scenarios/long_context.rs b/crates/switchyard-soak/src/scenarios/long_context.rs
new file mode 100644
index 000000000..6958e847f
--- /dev/null
+++ b/crates/switchyard-soak/src/scenarios/long_context.rs
@@ -0,0 +1,36 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::{
+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, token_text,
+ user,
+};
+
+pub fn build(options: ScenarioOptions<'_>) -> Scenario {
+ let mut lengths = vec![8_192, 32_000, options.context_window_tokens * 9 / 10];
+ lengths.retain(|length| *length < options.context_window_tokens);
+ lengths.sort_unstable();
+ lengths.dedup();
+ let sessions = lengths
+ .into_iter()
+ .map(|tokens| Session {
+ session_id: format!("long-context-{tokens}"),
+ payloads: vec![chat_payload(
+ options.model,
+ vec![user(format!(
+ "{}\nSummarize the final marker in one sentence.",
+ token_text(tokens, "long_context")
+ ))],
+ options.max_output_tokens.min(64),
+ )],
+ })
+ .collect();
+ Scenario::chat(
+ "long-context",
+ ScenarioGroup::Core,
+ "Requests at 8K, 32K, and near the configured context window.",
+ "TTFT rises with input length without routing errors or memory growth.",
+ ErrorExpectation::SUCCESS,
+ sessions,
+ )
+}
diff --git a/crates/switchyard-soak/src/scenarios/mixed_traffic.rs b/crates/switchyard-soak/src/scenarios/mixed_traffic.rs
new file mode 100644
index 000000000..ae1a444a4
--- /dev/null
+++ b/crates/switchyard-soak/src/scenarios/mixed_traffic.rs
@@ -0,0 +1,41 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::{
+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, token_text,
+ user,
+};
+
+pub fn build(options: ScenarioOptions<'_>) -> Scenario {
+ let lengths = [16, 24, 32, 48, 64, 96, 128, 1_024, 2_048, 8_192];
+ let sessions = lengths
+ .into_iter()
+ .enumerate()
+ .map(|(index, tokens)| Session {
+ session_id: format!("mixed-{index}"),
+ payloads: vec![chat_payload(
+ options.model,
+ vec![user(format!(
+ "{}\nReply with the request number {index}.",
+ token_text(
+ tokens.min(options.context_window_tokens / 2),
+ "mixed_traffic"
+ )
+ ))],
+ if index == 9 {
+ 256
+ } else {
+ options.max_output_tokens
+ },
+ )],
+ })
+ .collect();
+ Scenario::chat(
+ "mixed-traffic",
+ ScenarioGroup::Core,
+ "A 70/20/10 mix of short, medium, and long requests.",
+ "Tail latency stays bounded when unlike request sizes share one route.",
+ ErrorExpectation::SUCCESS,
+ sessions,
+ )
+}
diff --git a/crates/switchyard-soak/src/scenarios/mod.rs b/crates/switchyard-soak/src/scenarios/mod.rs
new file mode 100644
index 000000000..b7bf9a2cf
--- /dev/null
+++ b/crates/switchyard-soak/src/scenarios/mod.rs
@@ -0,0 +1,528 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Reproducible request scenarios shared by the soak command and AIPerf runner.
+
+mod classifier_mix;
+mod client_cancellation;
+mod context_overflow;
+mod decode_heavy;
+mod failure_pressure;
+mod growing_conversation;
+mod large_tool_catalog;
+mod long_context;
+mod mixed_traffic;
+mod prefix_reuse;
+mod short_interactive;
+mod stage_transitions;
+mod tool_call_burst;
+
+use std::collections::HashSet;
+use std::fs;
+use std::path::{Path, PathBuf};
+
+use clap::ValueEnum;
+use serde::Serialize;
+use serde_json::{Value, json};
+
+use crate::client::Endpoint;
+
+/// Largest accepted context-window input for generated scenario data.
+const MAX_CONTEXT_WINDOW_TOKENS: usize = 1_000_000;
+const MIN_CONTEXT_WINDOW_TOKENS: usize = 16_384;
+const MAX_PROMPT_BYTES: usize = 1_000_000;
+
+/// Scenario family used by CLI selection and report grouping.
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ScenarioGroup {
+ Core,
+ Agentic,
+ Resilience,
+}
+
+/// Accepted client-visible error-rate range for one scenario.
+#[derive(Clone, Copy, Serialize)]
+pub struct ErrorExpectation {
+ pub min_rate: f64,
+ pub max_rate: f64,
+}
+
+impl ErrorExpectation {
+ pub const SUCCESS: Self = Self {
+ min_rate: 0.0,
+ max_rate: 0.0,
+ };
+ pub const MIXED: Self = Self {
+ min_rate: 0.01,
+ max_rate: 0.75,
+ };
+ pub const ALL: Self = Self {
+ min_rate: 0.8,
+ max_rate: 1.0,
+ };
+}
+
+/// Named scenario collection selected by the soak and benchmark commands.
+#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)]
+#[value(rename_all = "snake_case")]
+pub enum ScenarioSet {
+ Core,
+ Agentic,
+ Resilience,
+ Standard,
+ All,
+}
+
+impl ScenarioSet {
+ fn contains(self, group: ScenarioGroup) -> bool {
+ match self {
+ Self::Core => group == ScenarioGroup::Core,
+ Self::Agentic => group == ScenarioGroup::Agentic,
+ Self::Resilience => group == ScenarioGroup::Resilience,
+ Self::Standard => group != ScenarioGroup::Resilience,
+ Self::All => true,
+ }
+ }
+}
+
+/// Request-generation limits shared by every scenario builder.
+#[derive(Clone, Copy)]
+pub struct ScenarioOptions<'a> {
+ pub model: &'a str,
+ pub prompt_bytes: usize,
+ pub max_output_tokens: u32,
+ pub context_window_tokens: usize,
+}
+
+impl ScenarioOptions<'_> {
+ pub fn validate(&self) -> Result<(), String> {
+ if self.model.is_empty() {
+ return Err("scenario model must not be empty".to_string());
+ }
+ if self.prompt_bytes == 0 || self.max_output_tokens == 0 {
+ return Err("scenario prompt and output limits must be greater than zero".to_string());
+ }
+ if !(MIN_CONTEXT_WINDOW_TOKENS..=MAX_CONTEXT_WINDOW_TOKENS)
+ .contains(&self.context_window_tokens)
+ {
+ return Err(format!(
+ "--context-window-tokens must be between {MIN_CONTEXT_WINDOW_TOKENS} and {MAX_CONTEXT_WINDOW_TOKENS}"
+ ));
+ }
+ let prompt_limit = MAX_PROMPT_BYTES.min(self.context_window_tokens.saturating_mul(2));
+ if self.prompt_bytes > prompt_limit {
+ return Err(format!(
+ "--prompt-bytes must not exceed {prompt_limit} for this context window"
+ ));
+ }
+ Ok(())
+ }
+}
+
+/// One AIPerf conversation; payloads remain ordered within the session.
+#[derive(Clone, Serialize)]
+pub struct Session {
+ pub session_id: String,
+ pub payloads: Vec<Value>,
+}
+
+/// One request the Rust soak command can send.
+pub struct RequestCase {
+ pub endpoint: Endpoint,
+ pub session_id: String,
+ streaming_body: Value,
+ nonstreaming_body: Value,
+}
+
+impl RequestCase {
+ pub fn new(endpoint: Endpoint, session_id: String, streaming_body: Value) -> Self {
+ let mut nonstreaming_body = streaming_body.clone();
+ nonstreaming_body["stream"] = Value::Bool(false);
+ Self {
+ endpoint,
+ session_id,
+ streaming_body,
+ nonstreaming_body,
+ }
+ }
+}
+
+pub struct PreparedRequest<'a> {
+ pub endpoint: Endpoint,
+ pub session_id: &'a str,
+ pub body: &'a Value,
+}
+
+/// QPS point expressed relative to the benchmark command's base request rate.
+#[derive(Clone, Serialize)]
+pub struct RatePoint {
+ pub time_s: u32,
+ pub rate_multiplier: f64,
+}
+
+/// AIPerf schedule applied to a request scenario.
+#[derive(Clone, Serialize)]
+#[serde(tag = "kind", rename_all = "snake_case")]
+pub enum LoadProfile {
+ Fixed {
+ id: &'static str,
+ },
+ ConcurrencyKnee {
+ id: &'static str,
+ concurrency_steps: Vec<usize>,
+ },
+ TrafficBurst {
+ id: &'static str,
+ duration_seconds: u32,
+ points: Vec<RatePoint>,
+ },
+}
+
+/// Immutable scenario data built once before load workers start.
+pub struct Scenario {
+ pub id: &'static str,
+ pub group: ScenarioGroup,
+ pub description: &'static str,
+ pub expected: &'static str,
+ pub expected_error_rate: ErrorExpectation,
+ pub sessions: Vec<Session>,
+ pub soak_requests: Vec<RequestCase>,
+ pub load_profiles: Vec<LoadProfile>,
+}
+
+impl Scenario {
+ /// Build a Chat Completions scenario whose AIPerf payloads also drive soak traffic.
+ pub fn chat(
+ id: &'static str,
+ group: ScenarioGroup,
+ description: &'static str,
+ expected: &'static str,
+ expected_error_rate: ErrorExpectation,
+ sessions: Vec<Session>,
+ ) -> Self {
+ let soak_requests =
+ sessions
+ .iter()
+ .flat_map(|session| {
+ session.payloads.iter().cloned().map(|body| {
+ RequestCase::new(Endpoint::Chat, session.session_id.clone(), body)
+ })
+ })
+ .collect();
+ Self {
+ id,
+ group,
+ description,
+ expected,
+ expected_error_rate,
+ sessions,
+ soak_requests,
+ load_profiles: fixed_load(),
+ }
+ }
+
+ /// Return one prebuilt request with only its streaming flag changed.
+ pub fn request(&self, index: usize, stream: bool) -> PreparedRequest<'_> {
+ let selected = &self.soak_requests[index % self.soak_requests.len()];
+ PreparedRequest {
+ endpoint: selected.endpoint,
+ session_id: &selected.session_id,
+ body: if stream {
+ &selected.streaming_body
+ } else {
+ &selected.nonstreaming_body
+ },
+ }
+ }
+}
+
+/// Return the one fixed schedule used by ordinary request scenarios.
+pub fn fixed_load() -> Vec<LoadProfile> {
+ vec![LoadProfile::Fixed { id: "fixed" }]
+}
+
+/// Build the three bounded load schedules used for the short baseline.
+pub fn baseline_load() -> Vec<LoadProfile> {
+ vec![
+ LoadProfile::Fixed { id: "fixed" },
+ LoadProfile::ConcurrencyKnee {
+ id: "concurrency-knee",
+ concurrency_steps: vec![1, 4, 16, 64, 128],
+ },
+ LoadProfile::TrafficBurst {
+ id: "traffic-burst",
+ duration_seconds: 30,
+ points: vec![
+ RatePoint {
+ time_s: 0,
+ rate_multiplier: 1.0,
+ },
+ RatePoint {
+ time_s: 10,
+ rate_multiplier: 1.0,
+ },
+ RatePoint {
+ time_s: 11,
+ rate_multiplier: 10.0,
+ },
+ RatePoint {
+ time_s: 16,
+ rate_multiplier: 10.0,
+ },
+ RatePoint {
+ time_s: 17,
+ rate_multiplier: 1.0,
+ },
+ RatePoint {
+ time_s: 30,
+ rate_multiplier: 1.0,
+ },
+ ],
+ },
+ ]
+}
+
+/// Build one raw Chat Completions payload.
+pub fn chat_payload(model: &str, messages: Vec<Value>, max_tokens: u32) -> Value {
+ json!({
+ "model": model,
+ "messages": messages,
+ "max_tokens": max_tokens,
+ "temperature": 0,
+ "stream": true,
+ })
+}
+
+/// Build one user message.
+pub fn user(content: impl Into<String>) -> Value {
+ json!({"role": "user", "content": content.into()})
+}
+
+/// Build one assistant message.
+pub fn assistant(content: impl Into<String>) -> Value {
+ json!({"role": "assistant", "content": content.into()})
+}
+
+/// Build repeated context with a stable approximate token count.
+pub fn token_text(tokens: usize, label: &str) -> String {
+ format!("[scenario:{label}] {}", "x ".repeat(tokens))
+}
+
+/// Build the complete catalog in stable report order.
+pub fn catalog(options: ScenarioOptions<'_>) -> Result<Vec<Scenario>, String> {
+ options.validate()?;
+ Ok(vec![
+ short_interactive::build(options),
+ long_context::build(options),
+ decode_heavy::build(options),
+ prefix_reuse::build(options),
+ mixed_traffic::build(options),
+ growing_conversation::build(options),
+ large_tool_catalog::build(options),
+ tool_call_burst::build(options),
+ stage_transitions::build(options),
+ classifier_mix::build(options),
+ context_overflow::build(options),
+ failure_pressure::build(options),
+ client_cancellation::build(options),
+ ])
+}
+
+/// Select explicit scenario IDs or every scenario in a named set.
+pub fn select(
+ options: ScenarioOptions<'_>,
+ set: ScenarioSet,
+ requested: &[String],
+) -> Result<Vec<Scenario>, String> {
+ let mut catalog = catalog(options)?;
+ if requested.is_empty() {
+ return Ok(catalog
+ .into_iter()
+ .filter(|scenario| set.contains(scenario.group))
+ .collect());
+ }
+
+ let mut seen = HashSet::new();
+ let mut selected = Vec::new();
+ for id in requested {
+ if !seen.insert(id) {
+ return Err(format!("scenario {id:?} was requested more than once"));
+ }
+ let index = catalog
+ .iter()
+ .position(|scenario| scenario.id == id)
+ .ok_or_else(|| format!("unknown scenario {id:?}"))?;
+ selected.push(catalog.remove(index));
+ }
+ Ok(selected)
+}
+
+#[derive(Serialize)]
+struct InputsFile<'a> {
+ data: &'a [Session],
+}
+
+#[derive(Serialize)]
+struct Manifest<'a> {
+ schema_version: u32,
+ model: &'a str,
+ scenarios: Vec<ManifestScenario<'a>>,
+}
+
+#[derive(Serialize)]
+struct ManifestScenario<'a> {
+ id: &'a str,
+ group: ScenarioGroup,
+ description: &'a str,
+ expected: &'a str,
+ expected_error_rate: ErrorExpectation,
+ input_file: String,
+ load_profiles: &'a [LoadProfile],
+}
+
+/// Export the selected scenario catalog in AIPerf's verbatim inputs JSON format.
+pub fn export(scenarios: &[Scenario], model: &str, output_dir: &Path) -> Result<PathBuf, String> {
+ if output_dir.exists() {
+ return Err(format!(
+ "scenario output directory already exists: {}",
+ output_dir.display()
+ ));
+ }
+ let inputs_dir = output_dir.join("inputs");
+ fs::create_dir_all(&inputs_dir).map_err(|error| error.to_string())?;
+
+ let mut entries = Vec::new();
+ for scenario in scenarios {
+ let relative = format!("inputs/{}.json", scenario.id);
+ let path = output_dir.join(&relative);
+ let body = serde_json::to_string_pretty(&InputsFile {
+ data: &scenario.sessions,
+ })
+ .map_err(|error| error.to_string())?;
+ fs::write(path, format!("{body}\n")).map_err(|error| error.to_string())?;
+ entries.push(ManifestScenario {
+ id: scenario.id,
+ group: scenario.group,
+ description: scenario.description,
+ expected: scenario.expected,
+ expected_error_rate: scenario.expected_error_rate,
+ input_file: relative,
+ load_profiles: &scenario.load_profiles,
+ });
+ }
+
+ let manifest_path = output_dir.join("manifest.json");
+ let body = serde_json::to_string_pretty(&Manifest {
+ schema_version: 1,
+ model,
+ scenarios: entries,
+ })
+ .map_err(|error| error.to_string())?;
+ fs::write(&manifest_path, format!("{body}\n")).map_err(|error| error.to_string())?;
+ Ok(manifest_path)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn options() -> ScenarioOptions<'static> {
+ ScenarioOptions {
+ model: "switchyard/test",
+ prompt_bytes: 128,
+ max_output_tokens: 32,
+ context_window_tokens: 16_384,
+ }
+ }
+
+ #[test]
+ fn catalog_is_unique_bounded_and_serializable() -> Result<(), String> {
+ let scenarios = catalog(options())?;
+ let names = scenarios
+ .iter()
+ .map(|scenario| scenario.id)
+ .collect::<HashSet<_>>();
+ assert_eq!(names.len(), scenarios.len());
+ assert!(scenarios.iter().all(|scenario| {
+ !scenario.sessions.is_empty()
+ && !scenario.soak_requests.is_empty()
+ && scenario.sessions.len() <= 32
+ && scenario
+ .sessions
+ .iter()
+ .all(|session| !session.payloads.is_empty() && session.payloads.len() <= 32)
+ }));
+
+ let parent = tempfile::tempdir().map_err(|error| error.to_string())?;
+ let output = parent.path().join("export");
+ let manifest = export(&scenarios, options().model, &output)?;
+ let payload: Value =
+ serde_json::from_str(&fs::read_to_string(manifest).map_err(|error| error.to_string())?)
+ .map_err(|error| error.to_string())?;
+ assert_eq!(payload["scenarios"].as_array().map(Vec::len), Some(13));
+ Ok(())
+ }
+
+ #[test]
+ fn explicit_selection_keeps_order_and_rejects_bad_names() -> Result<(), String> {
+ let requested = vec!["tool-call-burst".to_string(), "long-context".to_string()];
+ let selected = select(options(), ScenarioSet::Core, &requested)?;
+ assert_eq!(
+ selected
+ .iter()
+ .map(|scenario| scenario.id)
+ .collect::<Vec<_>>(),
+ ["tool-call-burst", "long-context"]
+ );
+ assert!(select(options(), ScenarioSet::Core, &["missing".to_string()]).is_err());
+ Ok(())
+ }
+
+ #[test]
+ fn agentic_payloads_keep_their_bounded_contracts() -> Result<(), String> {
+ let scenarios = catalog(options())?;
+ let find = |id| {
+ scenarios
+ .iter()
+ .find(|scenario| scenario.id == id)
+ .ok_or_else(|| format!("missing scenario {id}"))
+ };
+
+ let long = find("long-context")?;
+ assert!(long.sessions.iter().any(|session| {
+ session.payloads[0]["messages"][0]["content"]
+ .as_str()
+ .is_some_and(|content| content.len() > 8_000)
+ }));
+
+ let tools = find("large-tool-catalog")?;
+ assert_eq!(
+ tools.sessions[1].payloads[0]["tools"]
+ .as_array()
+ .map(Vec::len),
+ Some(64)
+ );
+
+ let burst = find("tool-call-burst")?;
+ assert_eq!(burst.sessions[0].payloads.len(), 9);
+ let final_messages = burst.sessions[0].payloads[8]["messages"]
+ .as_array()
+ .ok_or_else(|| "tool-call burst messages were not an array".to_string())?;
+ assert!(
+ final_messages.iter().any(|message| {
+ message["role"] == "tool" && message["tool_call_id"] == "call_7"
+ })
+ );
+
+ let growing = find("growing-conversation")?;
+ let message_counts = growing.sessions[0]
+ .payloads
+ .iter()
+ .map(|payload| payload["messages"].as_array().map(Vec::len).unwrap_or(0))
+ .collect::<Vec<_>>();
+ assert!(message_counts.windows(2).all(|pair| pair[0] < pair[1]));
+ assert!(find("failure-pressure")?.expected_error_rate.min_rate > 0.0);
+ Ok(())
+ }
+}
diff --git a/crates/switchyard-soak/src/scenarios/prefix_reuse.rs b/crates/switchyard-soak/src/scenarios/prefix_reuse.rs
new file mode 100644
index 000000000..7272ec501
--- /dev/null
+++ b/crates/switchyard-soak/src/scenarios/prefix_reuse.rs
@@ -0,0 +1,37 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::{
+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, token_text,
+ user,
+};
+
+pub fn build(options: ScenarioOptions<'_>) -> Scenario {
+ let prefix_tokens = (options.prompt_bytes / 2).clamp(512, 8_192);
+ let shared = token_text(prefix_tokens, "prefix_reuse_shared");
+ let sessions = (0..8)
+ .map(|index| {
+ let prefix = if index < 4 {
+ shared.clone()
+ } else {
+ token_text(prefix_tokens, &format!("prefix_reuse_unique_{index}"))
+ };
+ Session {
+ session_id: format!("prefix-reuse-{index}"),
+ payloads: vec![chat_payload(
+ options.model,
+ vec![user(format!("{prefix}\nReturn item {index}."))],
+ options.max_output_tokens.min(32),
+ )],
+ }
+ })
+ .collect();
+ Scenario::chat(
+ "prefix-reuse",
+ ScenarioGroup::Core,
+ "Matched requests with shared and unique long prefixes.",
+ "The report exposes cache-sensitive TTFT without changing route behavior.",
+ ErrorExpectation::SUCCESS,
+ sessions,
+ )
+}
diff --git a/crates/switchyard-soak/src/scenarios/short_interactive.rs b/crates/switchyard-soak/src/scenarios/short_interactive.rs
new file mode 100644
index 000000000..023f04161
--- /dev/null
+++ b/crates/switchyard-soak/src/scenarios/short_interactive.rs
@@ -0,0 +1,61 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::client::{Endpoint, request_body};
+
+use super::{
+ ErrorExpectation, RequestCase, Scenario, ScenarioGroup, ScenarioOptions, Session,
+ baseline_load, chat_payload, user,
+};
+
+pub fn build(options: ScenarioOptions<'_>) -> Scenario {
+ let prompts = [
+ "Reply with exactly OK.".to_string(),
+ "Name one primary color.".to_string(),
+ "Return the number after 41.".to_string(),
+ format!(
+ "[scenario:short_interactive] {} Reply briefly.",
+ "stable prefix ".repeat(options.prompt_bytes.div_ceil(14))
+ ),
+ ];
+ let sessions = prompts
+ .iter()
+ .enumerate()
+ .map(|(index, prompt)| Session {
+ session_id: format!("short-{index}"),
+ payloads: vec![chat_payload(
+ options.model,
+ vec![user(prompt)],
+ options.max_output_tokens,
+ )],
+ })
+ .collect();
+ let soak_requests = Endpoint::ALL
+ .iter()
+ .flat_map(|endpoint| {
+ prompts.iter().enumerate().map(move |(index, prompt)| {
+ RequestCase::new(
+ *endpoint,
+ format!("short-{index}-{}", endpoint.as_str()),
+ request_body(
+ *endpoint,
+ options.model,
+ prompt,
+ options.max_output_tokens,
+ true,
+ ),
+ )
+ })
+ })
+ .collect();
+ Scenario {
+ id: "short-interactive",
+ group: ScenarioGroup::Core,
+ description: "Short prompts that establish HTTP, routing, TTFT, and latency overhead.",
+ expected: "All public endpoints succeed; oha and AIPerf establish the baseline.",
+ expected_error_rate: ErrorExpectation::SUCCESS,
+ sessions,
+ soak_requests,
+ load_profiles: baseline_load(),
+ }
+}
diff --git a/crates/switchyard-soak/src/scenarios/stage_transitions.rs b/crates/switchyard-soak/src/scenarios/stage_transitions.rs
new file mode 100644
index 000000000..63916e37e
--- /dev/null
+++ b/crates/switchyard-soak/src/scenarios/stage_transitions.rs
@@ -0,0 +1,53 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use serde_json::json;
+
+use super::{
+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user,
+};
+
+pub fn build(options: ScenarioOptions<'_>) -> Scenario {
+ let mut messages = vec![user(
+ "[scenario:stage_exploration] Find the failing module, then fix it.",
+ )];
+ let mut payloads = vec![chat_payload(
+ options.model,
+ messages.clone(),
+ options.max_output_tokens.min(128),
+ )];
+
+ messages.extend([
+ json!({"role":"assistant","tool_calls":[{"id":"stage_1","type":"function","function":{"name":"run_tests","arguments":"{}"}}]}),
+ json!({"role":"tool","tool_call_id":"stage_1","content":"fatal: critical tool failure: repository index is unavailable"}),
+ ]);
+ payloads.push(chat_payload(
+ options.model,
+ messages.clone(),
+ options.max_output_tokens.min(128),
+ ));
+
+ messages.extend([
+ json!({"role":"assistant","tool_calls":[{"id":"stage_2","type":"function","function":{"name":"apply_patch","arguments":"{}"}}]}),
+ json!({"role":"tool","tool_call_id":"stage_2","content":"patch applied"}),
+ json!({"role":"assistant","tool_calls":[{"id":"stage_3","type":"function","function":{"name":"run_tests","arguments":"{}"}}]}),
+ json!({"role":"tool","tool_call_id":"stage_3","content":"all tests passed"}),
+ ]);
+ payloads.push(chat_payload(
+ options.model,
+ messages,
+ options.max_output_tokens.min(128),
+ ));
+
+ Scenario::chat(
+ "stage-transitions",
+ ScenarioGroup::Agentic,
+ "One growing session that moves through exploration, critical failure, and productive work.",
+ "The stage router changes tiers at the configured evidence boundaries.",
+ ErrorExpectation::SUCCESS,
+ vec![Session {
+ session_id: "stage-transitions".to_string(),
+ payloads,
+ }],
+ )
+}
diff --git a/crates/switchyard-soak/src/scenarios/tool_call_burst.rs b/crates/switchyard-soak/src/scenarios/tool_call_burst.rs
new file mode 100644
index 000000000..1c6b35d69
--- /dev/null
+++ b/crates/switchyard-soak/src/scenarios/tool_call_burst.rs
@@ -0,0 +1,65 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use serde_json::{Value, json};
+
+use super::{
+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user,
+};
+
+pub fn build(options: ScenarioOptions<'_>) -> Scenario {
+ let mut messages = vec![user(
+ "[scenario:tool_call_burst] Inspect eight shards, one call at a time.",
+ )];
+ let mut payloads = vec![chat_payload(
+ options.model,
+ messages.clone(),
+ options.max_output_tokens.min(128),
+ )];
+ for index in 0..8 {
+ let call_id = format!("call_{index}");
+ messages.push(json!({
+ "role": "assistant",
+ "content": null,
+ "tool_calls": [{
+ "id": call_id,
+ "type": "function",
+ "function": {"name": "inspect_shard", "arguments": format!("{{\"shard\":{index}}}")}
+ }]
+ }));
+ messages.push(json!({
+ "role": "tool",
+ "tool_call_id": format!("call_{index}"),
+ "content": format!("shard {index}: healthy")
+ }));
+ payloads.push(chat_payload(
+ options.model,
+ messages.clone(),
+ options.max_output_tokens.min(128),
+ ));
+ }
+ for payload in &mut payloads {
+ payload["tools"] = Value::Array(vec![json!({
+ "type": "function",
+ "function": {
+ "name": "inspect_shard",
+ "parameters": {
+ "type": "object",
+ "properties": {"shard": {"type": "integer"}},
+ "required": ["shard"]
+ }
+ }
+ })]);
+ }
+ Scenario::chat(
+ "tool-call-burst",
+ ScenarioGroup::Agentic,
+ "An eight-turn burst of linked assistant tool calls and tool results.",
+ "The route keeps session state and forwards every call/result pair.",
+ ErrorExpectation::SUCCESS,
+ vec![Session {
+ session_id: "tool-call-burst".to_string(),
+ payloads,
+ }],
+ )
+}
diff --git a/crates/switchyard-soak/src/stats.rs b/crates/switchyard-soak/src/stats.rs
new file mode 100644
index 000000000..dcb8f1f20
--- /dev/null
+++ b/crates/switchyard-soak/src/stats.rs
@@ -0,0 +1,453 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Bounded in-memory run state, latency percentiles, and the final pass/fail summary.
+
+use std::collections::BTreeMap;
+use std::time::SystemTime;
+
+use rand::rngs::StdRng;
+use rand::{RngExt, SeedableRng};
+use serde::Serialize;
+
+/// Cap on retained latency samples; a long run stays within this bound by reservoir sampling.
+const RESERVOIR_SIZE: usize = 100_000;
+
+/// Round to three decimals so result files stay readable.
+pub fn round3(value: f64) -> f64 {
+ (value * 1000.0).round() / 1000.0
+}
+
+/// Parse a duration such as `30s`, `15m`, or `48h` into seconds.
+pub fn parse_duration(value: &str) -> Result<f64, String> {
+ let text = value.trim().to_lowercase();
+ let bad = || "duration must use s, m, or h, for example 30s or 48h".to_string();
+ let unit = text.chars().last().ok_or_else(bad)?;
+ let multiplier = match unit {
+ 's' => 1.0,
+ 'm' => 60.0,
+ 'h' => 3600.0,
+ _ => return Err(bad()),
+ };
+ let number = &text[..text.len() - unit.len_utf8()];
+ let seconds = number.parse::<f64>().map_err(|_| bad())? * multiplier;
+ if !seconds.is_finite() || seconds <= 0.0 {
+ return Err("duration must be greater than zero".to_string());
+ }
+ Ok(seconds)
+}
+
+#[derive(Default)]
+pub struct LatencyStats {
+ pub p50_ms: Option<f64>,
+ pub p95_ms: Option<f64>,
+ pub p99_ms: Option<f64>,
+ pub max_ms: Option<f64>,
+}
+
+fn percentile_sorted(values: &[f64], quantile: f64) -> Option<f64> {
+ let index = (((values.len().checked_sub(1)?) as f64) * quantile).round_ties_even() as usize;
+ Some(round3(values[index]))
+}
+
+pub fn latency_stats(values: &mut [f64]) -> LatencyStats {
+ values.sort_by(|a, b| a.total_cmp(b));
+ LatencyStats {
+ p50_ms: percentile_sorted(values, 0.50),
+ p95_ms: percentile_sorted(values, 0.95),
+ p99_ms: percentile_sorted(values, 0.99),
+ max_ms: values.last().copied().map(round3),
+ }
+}
+
+/// ISO-8601 UTC timestamp at second precision, e.g. `2026-07-30T18:04:00Z`.
+pub fn now_utc_string() -> String {
+ humantime::format_rfc3339_seconds(SystemTime::now()).to_string()
+}
+
+/// Compact UTC stamp for a results directory, e.g. `20260730T180400Z`.
+pub fn utc_dir_stamp() -> String {
+ now_utc_string()
+ .chars()
+ .filter(|character| !matches!(character, '-' | ':'))
+ .collect()
+}
+
+/// Request results collected since the previous report.
+#[derive(Default)]
+pub struct IntervalStats {
+ pub successes: u64,
+ pub failures: u64,
+ pub latencies_ms: Vec<f64>,
+}
+
+/// Bounded in-memory state for one soak run.
+pub struct RunStats {
+ rng: StdRng,
+ interval: IntervalStats,
+ pub total_successes: u64,
+ pub total_failures: u64,
+ pub endpoint_successes: BTreeMap<String, u64>,
+ pub endpoint_failures: BTreeMap<String, u64>,
+ pub scenario_successes: BTreeMap<String, u64>,
+ pub scenario_failures: BTreeMap<String, u64>,
+ pub error_kinds: BTreeMap<String, u64>,
+ latency_reservoir: Vec<f64>,
+ latency_count: u64,
+ pub health_checks: u64,
+ pub health_failures: u64,
+ pub metrics_checks: u64,
+ pub metrics_failures: u64,
+ pub process_checks: u64,
+ pub process_failures: u64,
+ pub canaries: u64,
+ pub canary_failures: u64,
+ pub server_restarts: u64,
+ pub previous_server_requests: Option<f64>,
+ pub rss_samples: Vec<f64>,
+ pub completed_duration: bool,
+}
+
+impl RunStats {
+ pub fn new(seed: u64) -> Self {
+ Self {
+ rng: StdRng::seed_from_u64(seed),
+ interval: IntervalStats::default(),
+ total_successes: 0,
+ total_failures: 0,
+ endpoint_successes: BTreeMap::new(),
+ endpoint_failures: BTreeMap::new(),
+ scenario_successes: BTreeMap::new(),
+ scenario_failures: BTreeMap::new(),
+ error_kinds: BTreeMap::new(),
+ latency_reservoir: Vec::new(),
+ latency_count: 0,
+ health_checks: 0,
+ health_failures: 0,
+ metrics_checks: 0,
+ metrics_failures: 0,
+ process_checks: 0,
+ process_failures: 0,
+ canaries: 0,
+ canary_failures: 0,
+ server_restarts: 0,
+ previous_server_requests: None,
+ rss_samples: Vec::new(),
+ completed_duration: false,
+ }
+ }
+
+ /// Record one completed inference request; `error_kind` is `None` on success.
+ pub fn record(
+ &mut self,
+ endpoint: &str,
+ scenario: &str,
+ latency_ms: f64,
+ error_kind: Option<&str>,
+ ) {
+ match error_kind {
+ None => {
+ self.interval.successes += 1;
+ self.total_successes += 1;
+ *self
+ .endpoint_successes
+ .entry(endpoint.to_string())
+ .or_default() += 1;
+ *self
+ .scenario_successes
+ .entry(scenario.to_string())
+ .or_default() += 1;
+ }
+ Some(kind) => {
+ self.interval.failures += 1;
+ self.total_failures += 1;
+ *self
+ .endpoint_failures
+ .entry(endpoint.to_string())
+ .or_default() += 1;
+ *self
+ .scenario_failures
+ .entry(scenario.to_string())
+ .or_default() += 1;
+ *self.error_kinds.entry(kind.to_string()).or_default() += 1;
+ }
+ }
+ self.interval.latencies_ms.push(latency_ms);
+ self.latency_count += 1;
+ if self.latency_reservoir.len() < RESERVOIR_SIZE {
+ self.latency_reservoir.push(latency_ms);
+ } else {
+ let index = self.rng.random_range(0..self.latency_count) as usize;
+ if index < RESERVOIR_SIZE {
+ self.latency_reservoir[index] = latency_ms;
+ }
+ }
+ }
+
+ /// Return and reset the current interval.
+ pub fn take_interval(&mut self) -> IntervalStats {
+ std::mem::take(&mut self.interval)
+ }
+}
+
+/// Final result written to `summary.json`.
+#[derive(Serialize)]
+pub struct Summary {
+ pub passed: bool,
+ pub failure_reasons: Vec<String>,
+ pub completed_duration: bool,
+ pub elapsed_seconds: f64,
+ pub requests: u64,
+ pub successes: u64,
+ pub failures: u64,
+ pub error_rate: f64,
+ pub requests_per_second: f64,
+ pub endpoint_successes: BTreeMap<String, u64>,
+ pub endpoint_failures: BTreeMap<String, u64>,
+ pub scenario_successes: BTreeMap<String, u64>,
+ pub scenario_failures: BTreeMap<String, u64>,
+ pub error_kinds: BTreeMap<String, u64>,
+ pub latency_p50_ms: Option<f64>,
+ pub latency_p95_ms: Option<f64>,
+ pub latency_p99_ms: Option<f64>,
+ pub health_checks: u64,
+ pub health_failures: u64,
+ pub metrics_checks: u64,
+ pub metrics_failures: u64,
+ pub process_checks: u64,
+ pub process_failures: u64,
+ pub invalid_request_canaries: u64,
+ pub invalid_request_canary_failures: u64,
+ pub detected_server_restarts: u64,
+ pub rss_first_mib: Option<f64>,
+ pub rss_last_mib: Option<f64>,
+ pub rss_max_mib: Option<f64>,
+ pub rss_growth_mib: Option<f64>,
+ pub error_records: u64,
+ pub dropped_error_records: u64,
+}
+
+/// Build the final result and its release-gate reasons.
+pub fn build_summary(
+ stats: &RunStats,
+ elapsed_seconds: f64,
+ max_error_rate: f64,
+ max_rss_growth_mib: Option<f64>,
+ error_records: u64,
+ dropped_error_records: u64,
+ task_failures: &[String],
+) -> Summary {
+ let total = stats.total_successes + stats.total_failures;
+ let error_rate = if total > 0 {
+ stats.total_failures as f64 / total as f64
+ } else {
+ 1.0
+ };
+ let rss_first = stats.rss_samples.first().copied();
+ let rss_last = stats.rss_samples.last().copied();
+ let rss_growth = match (rss_first, rss_last) {
+ (Some(first), Some(last)) => Some(last - first),
+ _ => None,
+ };
+
+ let mut reasons: Vec<String> = task_failures.to_vec();
+ if !stats.completed_duration {
+ reasons.push("the run stopped before the requested duration".to_string());
+ }
+ if total == 0 {
+ reasons.push("no inference requests completed".to_string());
+ }
+ if error_rate > max_error_rate {
+ reasons.push(format!(
+ "request error rate {:.4}% exceeded the {:.4}% limit",
+ error_rate * 100.0,
+ max_error_rate * 100.0
+ ));
+ }
+ if stats.health_failures > 0 {
+ reasons.push(format!("{} liveness checks failed", stats.health_failures));
+ }
+ if stats.metrics_failures > 0 {
+ reasons.push(format!(
+ "{} server metrics checks failed",
+ stats.metrics_failures
+ ));
+ }
+ if stats.process_failures > 0 {
+ reasons.push(format!(
+ "{} server process checks failed",
+ stats.process_failures
+ ));
+ }
+ if stats.canary_failures > 0 {
+ reasons.push(format!(
+ "{} invalid-request recovery checks failed",
+ stats.canary_failures
+ ));
+ }
+ if stats.server_restarts > 0 {
+ reasons.push(format!(
+ "server counters reset {} time(s)",
+ stats.server_restarts
+ ));
+ }
+ if let (Some(limit), Some(growth)) = (max_rss_growth_mib, rss_growth)
+ && growth > limit
+ {
+ reasons.push(format!(
+ "server RSS grew {growth:.1} MiB, above the {limit:.1} MiB limit"
+ ));
+ }
+
+ let rss_max = stats
+ .rss_samples
+ .iter()
+ .copied()
+ .fold(None, |acc: Option<f64>, value| {
+ Some(acc.map_or(value, |m| m.max(value)))
+ });
+ let requests_per_second = if elapsed_seconds > 0.0 {
+ total as f64 / elapsed_seconds
+ } else {
+ 0.0
+ };
+
+ let mut latency_samples = stats.latency_reservoir.clone();
+ let latency = latency_stats(&mut latency_samples);
+
+ Summary {
+ passed: reasons.is_empty(),
+ failure_reasons: reasons,
+ completed_duration: stats.completed_duration,
+ elapsed_seconds: round3(elapsed_seconds),
+ requests: total,
+ successes: stats.total_successes,
+ failures: stats.total_failures,
+ error_rate,
+ requests_per_second,
+ endpoint_successes: stats.endpoint_successes.clone(),
+ endpoint_failures: stats.endpoint_failures.clone(),
+ scenario_successes: stats.scenario_successes.clone(),
+ scenario_failures: stats.scenario_failures.clone(),
+ error_kinds: stats.error_kinds.clone(),
+ latency_p50_ms: latency.p50_ms,
+ latency_p95_ms: latency.p95_ms,
+ latency_p99_ms: latency.p99_ms,
+ health_checks: stats.health_checks,
+ health_failures: stats.health_failures,
+ metrics_checks: stats.metrics_checks,
+ metrics_failures: stats.metrics_failures,
+ process_checks: stats.process_checks,
+ process_failures: stats.process_failures,
+ invalid_request_canaries: stats.canaries,
+ invalid_request_canary_failures: stats.canary_failures,
+ detected_server_restarts: stats.server_restarts,
+ rss_first_mib: rss_first,
+ rss_last_mib: rss_last,
+ rss_max_mib: rss_max,
+ rss_growth_mib: rss_growth,
+ error_records,
+ dropped_error_records,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn parse_duration_accepts_units_and_rejects_invalid_values() {
+ assert_eq!(parse_duration("30s"), Ok(30.0));
+ assert_eq!(parse_duration(".5s"), Ok(0.5));
+ assert_eq!(parse_duration("2.5m"), Ok(150.0));
+ assert_eq!(parse_duration("48h"), Ok(172_800.0));
+ for value in ["0s", "10", "5x", "-3s", "1d", "NaNs", "infs"] {
+ assert!(parse_duration(value).is_err(), "{value} should be rejected");
+ }
+ }
+
+ #[test]
+ fn latency_stats_sorts_once_and_handles_empty_input() {
+ let mut values = [60.0, 10.0, 50.0, 20.0, 40.0, 30.0];
+ let latency = latency_stats(&mut values);
+ assert_eq!(latency.p50_ms, Some(30.0));
+ assert_eq!(latency.p95_ms, Some(60.0));
+ assert_eq!(latency.p99_ms, Some(60.0));
+ assert_eq!(latency.max_ms, Some(60.0));
+
+ let empty = latency_stats(&mut []);
+ assert_eq!(empty.p50_ms, None);
+ assert_eq!(empty.max_ms, None);
+ }
+
+ #[test]
+ fn record_tracks_interval_and_cumulative_results() {
+ let mut stats = RunStats::new(1);
+ stats.record("chat", "short-interactive", 10.0, None);
+ stats.record("messages", "long-context", 20.0, Some("timeout"));
+
+ let interval = stats.take_interval();
+ assert_eq!(interval.successes, 1);
+ assert_eq!(interval.failures, 1);
+ assert_eq!(stats.total_successes, 1);
+ assert_eq!(stats.total_failures, 1);
+ assert_eq!(stats.error_kinds.get("timeout"), Some(&1));
+ }
+
+ #[test]
+ fn latency_samples_stay_bounded_after_the_reservoir_fills() {
+ let mut stats = RunStats::new(1);
+ for sample in 0..=RESERVOIR_SIZE {
+ stats.record("chat", "short-interactive", sample as f64, None);
+ }
+
+ assert_eq!(stats.latency_reservoir.len(), RESERVOIR_SIZE);
+ assert_eq!(stats.latency_count, RESERVOIR_SIZE as u64 + 1);
+ }
+
+ #[test]
+ fn summary_reports_all_failed_gates() {
+ let mut stats = RunStats::new(1);
+ stats.total_successes = 998;
+ stats.total_failures = 2;
+ stats.health_failures = 1;
+ stats.metrics_failures = 1;
+ stats.process_failures = 1;
+ stats.canary_failures = 1;
+ stats.server_restarts = 1;
+ stats.rss_samples = vec![100.0, 700.0];
+
+ let summary = build_summary(
+ &stats,
+ 10.0,
+ 0.001,
+ Some(512.0),
+ 0,
+ 0,
+ &["reporter failed: boom".to_string()],
+ );
+
+ assert!(!summary.passed);
+ for reason in [
+ "run stopped",
+ "error rate",
+ "liveness",
+ "metrics",
+ "process",
+ "invalid-request",
+ "counters reset",
+ "RSS grew",
+ "reporter failed",
+ ] {
+ assert!(
+ summary
+ .failure_reasons
+ .iter()
+ .any(|message| message.contains(reason)),
+ "missing {reason:?}: {:?}",
+ summary.failure_reasons,
+ );
+ }
+ assert_eq!(summary.rss_growth_mib, Some(600.0));
+ }
+}
diff --git a/crates/switchyard-soak/tests/soak.rs b/crates/switchyard-soak/tests/soak.rs
new file mode 100644
index 000000000..26c8dcbda
--- /dev/null
+++ b/crates/switchyard-soak/tests/soak.rs
@@ -0,0 +1,331 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! End-to-end tests against a local server that implements the public Switchyard APIs.
+
+use std::collections::HashSet;
+use std::error::Error;
+use std::sync::Arc;
+
+use axum::Router;
+use axum::extract::{Json, State};
+use axum::http::{HeaderMap, StatusCode, header};
+use axum::response::{IntoResponse, Response};
+use axum::routing::{get, post};
+use clap::Parser;
+use parking_lot::Mutex;
+use serde_json::{Value, json};
+
+use switchyard_soak::{Args, run};
+
+type TestResult = Result<(), Box<dyn Error>>;
+
+struct TestServer {
+ base_url: String,
+ task: tokio::task::JoinHandle<()>,
+}
+
+impl Drop for TestServer {
+ fn drop(&mut self) {
+ self.task.abort();
+ }
+}
+
+async fn serve(app: Router) -> Result<TestServer, Box<dyn Error>> {
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
+ let addr = listener.local_addr()?;
+ let task = tokio::spawn(async move {
+ let _ = axum::serve(listener, app.into_make_service()).await;
+ });
+ Ok(TestServer {
+ base_url: format!("http://{addr}"),
+ task,
+ })
+}
+
+#[derive(Clone, Default)]
+struct MockState {
+ is_chat_broken: bool,
+ seen: Arc<Mutex<HashSet<String>>>,
+}
+
+impl MockState {
+ fn record(&self, endpoint: &str, body: &Value) {
+ let stream = body.get("stream").and_then(Value::as_bool) == Some(true);
+ self.seen.lock().insert(format!("{endpoint}:{stream}"));
+ }
+}
+
+fn switchyard(is_chat_broken: bool) -> (Router, Arc<Mutex<HashSet<String>>>) {
+ async fn health() -> Response {
+ Json(json!({"status": "ok"})).into_response()
+ }
+ async fn metrics() -> Response {
+ "switchyard_total_requests 1\nswitchyard_total_errors 0\n".into_response()
+ }
+ async fn models() -> Response {
+ Json(json!({"data": [{"id": "soak-route"}]})).into_response()
+ }
+ async fn chat(
+ State(state): State<MockState>,
+ headers: HeaderMap,
+ Json(body): Json<Value>,
+ ) -> Response {
+ let invalid_messages = body.get("messages").is_some_and(|messages| {
+ messages
+ .as_array()
+ .is_none_or(|messages| messages.is_empty())
+ });
+ if invalid_messages {
+ return if state.is_chat_broken {
+ Json(json!({"choices": []})).into_response()
+ } else {
+ (StatusCode::BAD_REQUEST, "messages must not be empty").into_response()
+ };
+ }
+ if !headers.contains_key("x-switchyard-session-id") {
+ return (StatusCode::BAD_REQUEST, "missing session id").into_response();
+ }
+ state.record("chat", &body);
+ if state.is_chat_broken {
+ return if body.get("stream") == Some(&Value::Bool(true)) {
+ Json(json!({"choices": []})).into_response()
+ } else {
+ Json(json!({})).into_response()
+ };
+ }
+ response(&body, "choices")
+ }
+ async fn messages(
+ State(state): State<MockState>,
+ headers: HeaderMap,
+ Json(body): Json<Value>,
+ ) -> Response {
+ if !headers.contains_key("x-switchyard-session-id") {
+ return (StatusCode::BAD_REQUEST, "missing session id").into_response();
+ }
+ state.record("messages", &body);
+ response(&body, "content")
+ }
+ async fn responses(
+ State(state): State<MockState>,
+ headers: HeaderMap,
+ Json(body): Json<Value>,
+ ) -> Response {
+ if !headers.contains_key("x-switchyard-session-id") {
+ return (StatusCode::BAD_REQUEST, "missing session id").into_response();
+ }
+ state.record("responses", &body);
+ response(&body, "output")
+ }
+ fn response(body: &Value, field: &str) -> Response {
+ if body.get("stream") == Some(&Value::Bool(true)) {
+ let stream = match field {
+ "choices" => "data: {\"choices\":[]}\n\ndata: [DONE]\n\n",
+ "content" => "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n",
+ "output" => {
+ "event: response.completed\ndata: {\"type\":\"response.completed\"}\n\n"
+ }
+ _ => "",
+ };
+ ([(header::CONTENT_TYPE, "text/event-stream")], stream).into_response()
+ } else {
+ let mut response = serde_json::Map::new();
+ response.insert(field.to_string(), json!([]));
+ Json(Value::Object(response)).into_response()
+ }
+ }
+
+ let state = MockState {
+ is_chat_broken,
+ ..MockState::default()
+ };
+ let seen = state.seen.clone();
+ let app = Router::new()
+ .route("/health", get(health))
+ .route("/metrics", get(metrics))
+ .route("/v1/models", get(models))
+ .route("/v1/chat/completions", post(chat))
+ .route("/v1/messages", post(messages))
+ .route("/v1/responses", post(responses))
+ .with_state(state);
+ (app, seen)
+}
+
+fn args(base_url: &str, results_dir: &str) -> Result<Args, clap::Error> {
+ Args::try_parse_from([
+ "switchyard-soak",
+ "--base-url",
+ base_url,
+ "--model",
+ "soak-route",
+ "--duration",
+ "0.8s",
+ "--concurrency",
+ "2",
+ "--report-interval",
+ "0.1",
+ "--invalid-canary-interval",
+ "0.1",
+ "--scenario",
+ "short-interactive",
+ "--results-dir",
+ results_dir,
+ ])
+}
+
+#[tokio::test]
+async fn short_run_exercises_every_response_variant_and_passes() -> TestResult {
+ let (app, seen) = switchyard(false);
+ let server = serve(app).await?;
+ let dir = tempfile::tempdir()?;
+ let results_dir = dir.path().join("soak-results");
+ let args = args(
+ &server.base_url,
+ results_dir.to_str().ok_or("non-utf8 results dir")?,
+ )?;
+
+ args.validate()?;
+ assert_eq!(run(args).await?, 0);
+
+ let summary: Value =
+ serde_json::from_str(&std::fs::read_to_string(results_dir.join("summary.json"))?)?;
+ assert_eq!(summary["passed"], json!(true));
+ assert!(summary["invalid_request_canaries"].as_u64().unwrap_or(0) > 0);
+ assert_eq!(
+ *seen.lock(),
+ HashSet::from([
+ "chat:false".to_string(),
+ "chat:true".to_string(),
+ "messages:false".to_string(),
+ "messages:true".to_string(),
+ "responses:false".to_string(),
+ "responses:true".to_string(),
+ ])
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn bad_responses_and_canary_write_a_failing_summary() -> TestResult {
+ let (app, _) = switchyard(true);
+ let server = serve(app).await?;
+ let dir = tempfile::tempdir()?;
+ let results_dir = dir.path().join("soak-results");
+ let args = args(
+ &server.base_url,
+ results_dir.to_str().ok_or("non-utf8 results dir")?,
+ )?;
+
+ assert_eq!(run(args).await?, 1);
+
+ let summary: Value =
+ serde_json::from_str(&std::fs::read_to_string(results_dir.join("summary.json"))?)?;
+ assert_eq!(summary["passed"], json!(false));
+ assert!(
+ summary["error_kinds"]["invalid_response"]
+ .as_u64()
+ .unwrap_or(0)
+ > 0
+ );
+ assert!(
+ summary["error_kinds"]["invalid_stream"]
+ .as_u64()
+ .unwrap_or(0)
+ > 0
+ );
+ assert!(
+ summary["invalid_request_canary_failures"]
+ .as_u64()
+ .unwrap_or(0)
+ > 0
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn nonbaseline_scenarios_all_cross_the_public_request_path() -> TestResult {
+ let (app, _) = switchyard(false);
+ let server = serve(app).await?;
+ let dir = tempfile::tempdir()?;
+ let results_dir = dir.path().join("scenario-results");
+ let args = Args::try_parse_from([
+ "switchyard-soak",
+ "--base-url",
+ &server.base_url,
+ "--model",
+ "soak-route",
+ "--duration",
+ "0.8s",
+ "--concurrency",
+ "4",
+ "--report-interval",
+ "0.2",
+ "--invalid-canary-interval",
+ "0",
+ "--scenario",
+ "long-context",
+ "--scenario",
+ "decode-heavy",
+ "--scenario",
+ "prefix-reuse",
+ "--scenario",
+ "mixed-traffic",
+ "--scenario",
+ "growing-conversation",
+ "--scenario",
+ "large-tool-catalog",
+ "--scenario",
+ "tool-call-burst",
+ "--scenario",
+ "stage-transitions",
+ "--scenario",
+ "classifier-mix",
+ "--results-dir",
+ results_dir.to_str().ok_or("non-utf8 results dir")?,
+ ])?;
+
+ assert_eq!(run(args).await?, 0);
+ let summary: Value =
+ serde_json::from_str(&std::fs::read_to_string(results_dir.join("summary.json"))?)?;
+ let successes = summary["scenario_successes"]
+ .as_object()
+ .ok_or("scenario_successes was not an object")?;
+ for scenario in [
+ "long-context",
+ "decode-heavy",
+ "prefix-reuse",
+ "mixed-traffic",
+ "growing-conversation",
+ "large-tool-catalog",
+ "tool-call-burst",
+ "stage-transitions",
+ "classifier-mix",
+ ] {
+ assert!(successes.contains_key(scenario), "missing {scenario}");
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn unknown_model_is_rejected_before_the_run_starts() -> TestResult {
+ let (app, _) = switchyard(false);
+ let server = serve(app).await?;
+ let dir = tempfile::tempdir()?;
+ let results_dir = dir.path().join("soak-results");
+ let args = Args::try_parse_from([
+ "switchyard-soak",
+ "--base-url",
+ &server.base_url,
+ "--model",
+ "missing",
+ "--duration",
+ "1s",
+ "--results-dir",
+ results_dir.to_str().ok_or("non-utf8 results dir")?,
+ ])?;
+
+ let error = run(args).await.unwrap_err();
+ assert!(error.contains("is not listed"), "{error}");
+ Ok(())
+}
diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
index 87dc58b6c..23fd5140c 100644
--- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
@@ -91,7 +91,13 @@ impl FormatCodec for AnthropicMessagesCodec {
content,
});
}
- if let Some(messages) = body.get("messages").and_then(Value::as_array) {
+ if let Some(messages) = body.get("messages") {
+ let messages = messages
+ .as_array()
+ .ok_or_else(|| TranslationError::InvalidType {
+ path: "$.messages".to_string(),
+ expected: "array",
+ })?;
let mut generated_id = 0;
for (index, message) in messages.iter().enumerate() {
let Some(message) = message.as_object() else {
diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
index d829ca316..4b0394f43 100644
--- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
@@ -72,7 +72,13 @@ impl FormatCodec for OpenAiChatCodec {
..LlmRequest::default()
};
- if let Some(messages) = body.get("messages").and_then(Value::as_array) {
+ if let Some(messages) = body.get("messages") {
+ let messages = messages
+ .as_array()
+ .ok_or_else(|| TranslationError::InvalidType {
+ path: "$.messages".to_string(),
+ expected: "array",
+ })?;
let mut generated_id = 0;
for (index, message) in messages.iter().enumerate() {
let Some(message) = message.as_object() else {
diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs
index e689e5b21..c754994ce 100644
--- a/crates/switchyard-translation/tests/request_translation.rs
+++ b/crates/switchyard-translation/tests/request_translation.rs
@@ -1782,6 +1782,18 @@ fn malformed_request_fields_are_rejected() {
json!({"model": "claude", "max_tokens": "8", "messages": []}),
"invalid value at $.max_tokens: expected a non-negative integer",
),
+ (
+ "OpenAI Chat string messages",
+ WireFormat::OpenAiChat,
+ json!({"model": "gpt", "messages": "invalid"}),
+ "expected array at $.messages",
+ ),
+ (
+ "Anthropic string messages",
+ WireFormat::AnthropicMessages,
+ json!({"model": "claude", "max_tokens": 8, "messages": "invalid"}),
+ "expected array at $.messages",
+ ),
(
"Responses boolean input",
WireFormat::OpenAiResponses,
diff --git a/docs/operations/soak_test.md b/docs/operations/soak_test.md
new file mode 100644
index 000000000..c0ce76cb4
--- /dev/null
+++ b/docs/operations/soak_test.md
@@ -0,0 +1,342 @@
+# Soak test a release candidate
+
+A Switchyard soak test sends sustained traffic through a release-candidate
+server long enough to expose failures that short tests miss. Run it for 48
+hours before code freeze when a release changes libsy, the Rust server,
+routing, streaming, translation, or server lifecycle behavior.
+
+The standard test sends closed-loop traffic through:
+
+- OpenAI Chat Completions (`/v1/chat/completions`)
+- Anthropic Messages (`/v1/messages`)
+- OpenAI Responses (`/v1/responses`)
+- streaming and non-streaming responses
+- short and long inputs, long outputs, shared prefixes, and mixed request sizes
+- growing conversations, large tool catalogs, tool-call bursts, and stage transitions
+- deterministic easy/hard classifier mixes
+
+The runner also checks `/health` and `/metrics` every minute. Every five
+minutes, it sends an invalid Chat Completions request, expects HTTP 400, and
+then confirms the server is still live.
+
+The scenario catalog covers these distinct pressure angles:
+
+| Scenario | Pressure angle | What to review |
+|---|---|---|
+| `short-interactive` | Short prompts under fixed load, a concurrency knee, or a 10x request-rate burst | HTTP ceiling, TTFT, routing overhead, and the saturation point |
+| `long-context` | 8K, 32K, and near-window inputs | TTFT, memory, and context-sensitive route failures |
+| `decode-heavy` | 512-token and 1,024-token output limits | ITL, output tokens/second, and stream stability |
+| `prefix-reuse` | Matched shared and unique long prefixes | Cache-sensitive TTFT and token accounting |
+| `mixed-traffic` | A 70/20/10 short, medium, and long mix | p99 latency and head-of-line effects |
+| `growing-conversation` | Eight cumulative conversation turns in one session | Affinity, history cost, and per-turn latency growth |
+| `large-tool-catalog` | 16-tool and 64-tool JSON-schema catalogs | Serialization/routing overhead and intact tool forwarding |
+| `tool-call-burst` | Eight linked assistant call/tool-result turns | Session continuity and burst handling |
+| `stage-transitions` | One growing history across exploration, critical failure, and productive work | Stage-router tier changes and scorer output |
+| `classifier-mix` | Deterministic 80/20 then 50/50 easy/hard requests | Target share, classifier calls/errors, and classifier latency |
+| `context-overflow` | One target rejects a near-window request | Fallback to another eligible target |
+| `failure-pressure` | Bounded 429, 500, malformed verdict, and truncated stream | Retry recovery, explicit terminal errors, and connection health |
+| `client-cancellation` | A client timeout during a delayed response | Teardown and recovery of later traffic |
+
+`standard` includes the core and agentic rows. Run `resilience` separately because expected
+failures should not be compared as throughput samples.
+
+## Prepare the server
+
+Run the exact commit, build, server config, backend, and model planned for the
+release. Do not use a development server in front of a different Switchyard
+build.
+
+Start the standalone server and its libsy algorithms:
+
+```bash
+cargo build --release -p switchyard-server
+target/release/switchyard-server --config release-routes.toml \
+ > switchyard-soak.log 2>&1 &
+SOAK_SERVER_PID=$!
+```
+
+Wait for `GET http://127.0.0.1:4000/health` to return HTTP 200 with the JSON
+body `{"status": "ok"}`; the runner requires both before it starts. Check
+`GET http://127.0.0.1:4000/v1/models` and choose the model id that represents
+the release workload. Pass that exact id with `--model`.
+
+Run the server and test from a dedicated host, job scheduler, or terminal
+multiplexer that will stay alive for the full test. Confirm that the host will
+not suspend or restart and has enough disk space for the server log.
+
+## Build the soak tester
+
+The soak tester is a Rust binary. Build it once from the same checkout as the
+release, then run it directly:
+
+```bash
+cargo build --release -p switchyard-soak
+```
+
+## Compare routing algorithm performance
+
+Use `scripts/benchmark_routing_algorithms.py` to compare routing algorithms under the same load.
+The command runs oha and AIPerf sequentially for every model id, then writes `report.md`,
+`report.csv`, `report.json`, and `routing-overhead.svg` beside both tools' raw results:
+
+```bash
+python3.12 scripts/benchmark_routing_algorithms.py \
+ --base-url http://127.0.0.1:4000 \
+ --direct-base-url http://127.0.0.1:8100 \
+ --direct-model mock/weak \
+ --model noop=switchyard/noop \
+ --model passthrough=switchyard/passthrough \
+ --model random=switchyard/random \
+ --model llm_classifier=switchyard/classifier \
+ --model stage_router=switchyard/stage \
+ --concurrency 100 \
+ --request-count 1000 \
+ --scenario-set standard \
+ --load-profile fixed \
+ --profile-runs 3 \
+ --backend-label "release model deployment"
+```
+
+`--direct-base-url` and `--direct-model` add an AIPerf arm that calls the backend without
+Switchyard. The report writes the end-to-end latency and throughput difference for every routed
+arm. That difference isolates Switchyard overhead only when the direct and routed arms use the
+same backend deployment, model, and response settings. Resilience scenarios have different
+failure semantics, so the script does not run or compare the direct arm for them. For an
+authenticated backend, pass the name of the key variable, not the key itself, with
+`--direct-api-key-env NVIDIA_API_KEY`.
+
+The Markdown report starts with one overhead row for each route and workload. Positive request or
+TTFT latency means the routed request took longer than the direct request. Negative request or
+token throughput means the routed path processed less work. The `standard` scenario set includes
+all non-resilience request patterns exported by the Rust crate: short and long contexts, long
+outputs, shared prefixes, mixed request sizes, growing conversations, large tool catalogs,
+tool-call bursts, stage changes, and easy/hard classifier mixes.
+
+When the run includes a direct backend, the Markdown report embeds `routing-overhead.svg` before
+the overhead table. The two annotated heatmaps show TTFT and output-token-throughput changes for
+every route and workload. Each cell shows the absolute change in milliseconds or tokens per second
+and the percent change. Red cells are worse than the direct backend, and blue cells are better.
+The script uses only the Python standard library to create the SVG, so the same benchmark command
+reproduces the plot without a plotting package.
+
+The Rust crate exports one AIPerf `inputs-json` file per scenario. oha reuses the first exported
+`short-interactive` payload as a non-streaming fixed body and reports the raw HTTP request rate and
+latency ceiling. AIPerf replays every selected streaming session and reports request latency, time
+to first token (TTFT), inter-token latency (ITL), request throughput, output-token throughput, and
+multi-run confidence intervals. The command saves `/v1/stats` before and after each scenario/load
+cell, so the routing calls and errors total all independent repetitions in that cell. The report
+also records selected-target shares, classifier calls/errors and latency, and mean routing
+overhead. It runs jobs sequentially because simultaneous tools would compete for the same server
+capacity.
+
+Use the load schedules after the fixed scenario comparison establishes a baseline:
+
+```bash
+python3.12 scripts/benchmark_routing_algorithms.py \
+ --base-url http://127.0.0.1:4000 \
+ --model random=switchyard/random \
+ --model classifier=switchyard/classifier \
+ --scenario short-interactive \
+ --load-profile concurrency-knee \
+ --load-profile traffic-burst \
+ --concurrency 128 \
+ --request-rate 20
+```
+
+The concurrency knee uses bounded steps up to `--concurrency`. The traffic burst holds the base
+request rate, raises it to 10 times that rate for five seconds, then returns to the base. Pass
+`--load-profile all` to run fixed, knee, and burst schedules. These schedules apply to the short
+baseline; they are not separate request scenarios.
+
+To isolate routing overhead, configure every route to use the same target deployment and keep the
+scenario manifest, concurrency, request count, and profile-run count fixed. Set `--tokenizer` to
+the real model tokenizer when exact token counts matter. AIPerf uses deterministic sessions and a
+fixed seed so each route receives the same requests.
+
+This comparison command does not start a backend or Switchyard. Point it at a running Switchyard
+server backed by real models to measure end-to-end TTFT and token throughput with real tokenization
+and generated tokens. Real-model runs cost tokens and include provider queuing and model variance,
+so use a dedicated deployment and repeat the run before treating a small difference as an
+algorithm effect. Use the local scenario backend first for deterministic routing correctness and
+overhead, then rerun the same manifest against real models for capacity claims.
+
+## Check the local server and load tools
+
+The local test uses a request-aware Axum backend from the soak crate. Build the server, soak tester,
+and scenario backend from the commit under test:
+
+```bash
+cargo build --release -p switchyard-server -p switchyard-soak \
+ --bins --example switchyard-soak-mock
+```
+
+Install [oha](https://github.com/hatoo/oha) and
+[NVIDIA AIPerf](https://docs.nvidia.com/aiperf/reference/command-line-options):
+
+```bash
+cargo install oha
+uv tool install --python 3.12 'aiperf==0.11.0'
+```
+
+The benchmark checks the AIPerf version before it creates an output directory. It runs each
+repetition in a separate AIPerf process and calculates confidence intervals from those independent
+runs. Each process starts one required record processor before profiling. The benchmark gives each
+process a deadline based on its request schedule, then stops the whole process group if that deadline
+expires. This prevents a startup failure from leaving a benchmark stuck at zero processed records.
+
+Then run the local test from the repository root:
+
+```bash
+python3.12 scripts/run_local_soak_test.py \
+ --duration 10s \
+ --concurrency 4 \
+ --request-count 100
+```
+
+The local backend waits 40 ms before the first token and 1 ms between output tokens by default.
+It streams the output limit from each Rust scenario, so decode-heavy requests produce 512 or 1,024
+tokens and AIPerf can measure TTFT, ITL, and token throughput separately. Use
+`--mock-latency-ms` and `--mock-token-latency-ms` to change that deterministic timing model.
+
+`--duration` controls how long the Rust soak tester runs. `--request-count` controls how many
+measured requests each load tool sends to each algorithm. `--help` explains every flag. Set
+`OHA_BIN`, `AIPERF_BIN`,
+`SWITCHYARD_SERVER_BIN`, `SWITCHYARD_SOAK_BIN`, or `SWITCHYARD_SOAK_MOCK_BIN` when a command is not
+on `PATH` or not under `target/release`.
+
+The script checks all five commands before starting a process. A missing command prints a warning,
+the build or install command, and the matching environment variable. Cargo builds the scenario
+backend, but it does not install oha or AIPerf during a normal build.
+
+The script gives each tool one job:
+
+| Tool | Job in the local test |
+|---|---|
+| Scenario backend | Returns local OpenAI-compatible responses, valid easy/hard classifier verdicts, and bounded failures with no provider cost. |
+| oha | Measures the non-streaming `short-interactive` HTTP baseline through every route. |
+| Python route checks | Sends one ordinary Chat Completions request through each configured route before load starts. |
+| AIPerf | Replays Rust-exported streaming sessions through every route and records LLM, token, response-time, and confidence results. |
+| Combined report | Joins scenario, load, oha, AIPerf, and routing-counter metrics in Markdown, CSV, JSON, and an overhead plot. It keeps resilience rows separate from throughput rows. |
+| `switchyard-soak` | Runs the standard scenario set while checking public API variants, server health, metrics, process use, and required results. |
+
+`scripts/local_soak_test.toml` exercises `noop`, `random`, `passthrough`, `llm_classifier`, and
+`stage_router`. It uses the accepted maximum retry count (10), a zero-weight random target, and the
+upper classifier and stage thresholds (1.0). Classifier affinity is disabled so every measured
+request includes the classifier call. The scenario backend returns `p_solve=1.0` for easy markers
+and `p_solve=0.1` for hard markers; with the configured threshold, those requests select the weak
+and strong targets respectively. The config is validated with
+`switchyard-server --dry-run` before either service starts.
+
+Run resilience cases separately so expected transport failures do not contaminate throughput
+comparisons:
+
+```bash
+python3.12 scripts/benchmark_routing_algorithms.py \
+ --base-url http://127.0.0.1:4000 \
+ --model classifier=switchyard/classifier \
+ --scenario-set resilience \
+ --load-profile fixed \
+ --profile-runs 1
+```
+
+`context-overflow` checks target fallback, `failure-pressure` injects bounded 429, 500, malformed
+classifier, and truncated-stream cases, and `client-cancellation` uses a one-second client timeout
+against a delayed response. Their expected error-rate ranges appear in the report's Resilience
+section, and the command fails after writing the report when any row misses its range. The local
+runner calls the scenario backend's `/reset` endpoint before each AIPerf cell so every algorithm
+receives the same transient-failure sequence. When you run the comparison command directly against
+that backend, pass `--scenario-backend-reset-url http://127.0.0.1:8100/reset` to preserve the same
+comparison.
+
+The runner does not add limits for target count or recent-turn history because the server does not
+limit them. Rust tests cover the real limits: 4,096 saved route assignments, 100,000 response-time
+samples, and 10,000 error records. Server tests cover invalid thresholds, bad classifier responses,
+missing stage-router request IDs, target errors, and retry count 11, which is one above the maximum.
+
+## Run the 48-hour test
+
+Choose concurrency from the release capacity plan. Increase it in short
+test runs until you find the highest expected steady load that remains below
+the backend's rate limit. Use that load for the 48-hour run. An overload test
+that spends most of its time throttled does not measure release stability.
+
+```bash
+./target/release/switchyard-soak \
+ --base-url http://127.0.0.1:4000 \
+ --model RELEASE_MODEL_ID \
+ --duration 48h \
+ --concurrency 16 \
+ --server-pid "$SOAK_SERVER_PID" \
+ --max-rss-growth-mib 512
+```
+
+The runner keeps 16 requests in flight until the test ends. This can generate
+large usage charges against a metered backend. Use a dedicated test deployment,
+estimate the request volume first with a short run, and get approval for any
+paid-provider cost.
+
+Use a five-minute run to confirm the route and result files:
+
+```bash
+./target/release/switchyard-soak \
+ --base-url http://127.0.0.1:4000 \
+ --model RELEASE_MODEL_ID \
+ --duration 5m \
+ --concurrency 4 \
+ --report-interval 10
+```
+
+If the Switchyard endpoint requires a bearer token, pass the environment
+variable name instead of putting the token on the command line:
+
+```bash
+export SWITCHYARD_SOAK_TOKEN="..."
+./target/release/switchyard-soak \
+ --api-key-env SWITCHYARD_SOAK_TOKEN \
+ --model RELEASE_MODEL_ID
+```
+
+## Pass criteria
+
+The command exits with status 0 only when:
+
+- the requested duration completes;
+- at least one inference request completes;
+- the inference error rate stays at or below `--max-error-rate` (default `0`,
+ which means no inference request may fail);
+- every periodic liveness check passes;
+- every `/metrics` read returns both Switchyard request counters;
+- every requested process sample returns RSS and CPU data;
+- every invalid-request recovery check passes;
+- the server request counter never resets; and
+- RSS growth stays within `--max-rss-growth-mib` when that limit is set.
+
+If the release plan permits transient failures from a remote provider, set an
+explicit error budget with `--max-error-rate`. Record the reason for that
+exception in the release record.
+
+The RSS limit is deployment-specific. Set it from an approved baseline for the
+same model, concurrency, and worker count. Omit `--server-pid` and
+`--max-rss-growth-mib` when the server runs on another host, then collect
+memory and restart data from that host's monitoring system.
+
+## Review the results
+
+Each run creates a timestamped directory under `soak-results/`:
+
+- `config.json` records the non-secret test inputs.
+- `intervals.csv` records request rate, errors, latency percentiles, health,
+ Switchyard counters, RSS, and CPU once per reporting interval. `cpu_percent`
+ is the `ps` lifetime-average CPU for the process, not the interval's usage, so
+ read it as a long-run average rather than a spike detector.
+- `errors.jsonl` records up to 10,000 request and canary failures.
+- `summary.json` records the final pass result and any failed gates.
+
+Tail the run log to monitor progress, cumulative error rate, health, RSS, and the
+`OK`, `DEGRADED`, or `STALLED` interval status while the test runs.
+
+Before approving the release, check `intervals.csv` for late failures, falling
+throughput, increasing p95 or p99 latency, and steady RSS growth. Compare the
+first and last several hours, not only the run-wide averages. Attach
+`summary.json`, the interval chart, the Switchyard log, the tested commit, and
+the server config to the release record.
diff --git a/mkdocs.yml b/mkdocs.yml
index fcf40f715..d934b61ef 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -32,6 +32,7 @@ nav:
- Advisor-Gate Routing: routing_algorithms/advisor_gate_routing.md
- Operations:
- Context-Window Handling: operations/context_window.md
+ - Soak Testing: operations/soak_test.md
- Reference:
- CLI Reference: cli_reference.md
- TOML Schema: reference/toml_schema.md
diff --git a/scripts/aiperf_runner.py b/scripts/aiperf_runner.py
new file mode 100644
index 000000000..f83165df1
--- /dev/null
+++ b/scripts/aiperf_runner.py
@@ -0,0 +1,220 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Run the supported AIPerf release without allowing its startup race to hang."""
+
+import json
+import math
+import os
+import signal
+import statistics
+import subprocess
+import time
+from collections.abc import Sequence
+from pathlib import Path
+
+SUPPORTED_AIPERF_VERSION = "0.11.0"
+PROCESS_GROUP_GRACE_SECONDS = 5
+_T_CRITICAL_95 = {
+ 2: 12.706,
+ 3: 4.303,
+ 4: 3.182,
+ 5: 2.776,
+ 6: 2.571,
+ 7: 2.447,
+ 8: 2.365,
+ 9: 2.306,
+ 10: 2.262,
+}
+_REQUIRED_STATISTICS = (
+ ("error_request_count", "avg"),
+ ("request_count", "avg"),
+ ("request_throughput", "avg"),
+)
+
+
+def validate_aiperf_version(binary: str) -> None:
+ """Reject AIPerf versions whose CLI or startup behavior is not covered."""
+ try:
+ result = subprocess.run(
+ [binary, "--version"],
+ capture_output=True,
+ text=True,
+ check=False,
+ timeout=10,
+ )
+ except (OSError, subprocess.TimeoutExpired) as error:
+ raise RuntimeError(f"could not read the AIPerf version from {binary}: {error}") from error
+ version = result.stdout.strip()
+ if result.returncode != 0 or version != SUPPORTED_AIPERF_VERSION:
+ found = version or result.stderr.strip() or "unknown"
+ raise RuntimeError(
+ f"AIPerf {found} is unsupported; install {SUPPORTED_AIPERF_VERSION} with: "
+ f"uv tool install --python 3.12 'aiperf=={SUPPORTED_AIPERF_VERSION}'"
+ )
+
+
+def _process_group_exists(process_group: int) -> bool:
+ try:
+ os.killpg(process_group, 0)
+ except ProcessLookupError:
+ return False
+ except PermissionError:
+ # A completed leader's process-group id can be reused by a process we do not own.
+ return False
+ return True
+
+
+def _stop_process_group(process: subprocess.Popen[bytes]) -> None:
+ """Stop AIPerf and every worker process it started."""
+ process_group = process.pid
+ try:
+ os.killpg(process_group, signal.SIGTERM)
+ except ProcessLookupError:
+ pass
+ except PermissionError as error:
+ if process.poll() is None:
+ raise RuntimeError(
+ f"could not stop running AIPerf process group {process_group}"
+ ) from error
+ return
+ deadline = time.monotonic() + PROCESS_GROUP_GRACE_SECONDS
+ while _process_group_exists(process_group) and time.monotonic() < deadline:
+ process.poll()
+ time.sleep(0.05)
+ if _process_group_exists(process_group):
+ try:
+ os.killpg(process_group, signal.SIGKILL)
+ except ProcessLookupError:
+ pass
+ except PermissionError as error:
+ if process.poll() is None:
+ raise RuntimeError(
+ f"could not kill running AIPerf process group {process_group}"
+ ) from error
+ try:
+ process.wait(timeout=PROCESS_GROUP_GRACE_SECONDS)
+ except subprocess.TimeoutExpired as error:
+ raise RuntimeError(f"could not reap AIPerf process {process.pid}") from error
+
+
+def run_profile(
+ command: Sequence[str],
+ log_path: Path,
+ artifact_dir: Path,
+ timeout_seconds: int,
+) -> Path:
+ """Run one bounded AIPerf process and return its verified export."""
+ artifact_dir.parent.mkdir(parents=True, exist_ok=True)
+ with log_path.open("wb") as log:
+ try:
+ process = subprocess.Popen(
+ [*command, "--artifact-dir", str(artifact_dir)],
+ stdout=log,
+ stderr=subprocess.STDOUT,
+ start_new_session=True,
+ )
+ except OSError as error:
+ raise RuntimeError(f"could not start AIPerf: {error}") from error
+ stopped = False
+ try:
+ try:
+ returncode = process.wait(timeout=timeout_seconds)
+ except subprocess.TimeoutExpired as error:
+ timeout_message = f"AIPerf exceeded its {timeout_seconds}-second run limit"
+ log.write(f"\n{timeout_message}.\n".encode())
+ log.flush()
+ _stop_process_group(process)
+ stopped = True
+ raise RuntimeError(f"{timeout_message}; see {log_path}") from error
+ finally:
+ if not stopped and (process.poll() is None or _process_group_exists(process.pid)):
+ _stop_process_group(process)
+ if returncode != 0:
+ raise RuntimeError(f"AIPerf failed with status {returncode}; see {log_path}")
+ export_path = artifact_dir / "profile_export_aiperf.json"
+ if not export_path.is_file():
+ raise RuntimeError(f"AIPerf did not write {export_path.name}; see {log_path}")
+ return export_path
+
+
+def _summary(values: Sequence[float], unit: str) -> dict[str, float | str | None]:
+ """Return the 95% confidence summary used by the benchmark report."""
+ count = len(values)
+ mean = statistics.fmean(values)
+ standard_deviation = statistics.stdev(values) if count > 1 else 0.0
+ standard_error = standard_deviation / math.sqrt(count)
+ critical = _T_CRITICAL_95.get(count)
+ margin = critical * standard_error if critical is not None else 0.0
+ return {
+ "mean": mean,
+ "std": standard_deviation,
+ "min": min(values),
+ "max": max(values),
+ "cv": standard_deviation / mean if mean else 0.0,
+ "se": standard_error,
+ "ci_low": mean - margin,
+ "ci_high": mean + margin,
+ "t_critical": critical,
+ "unit": unit,
+ }
+
+
+def aggregate_exports(exports: Sequence[Path], output_path: Path) -> Path:
+ """Combine independent AIPerf exports into the aggregate schema consumed by the report."""
+ if not 2 <= len(exports) <= 10:
+ raise RuntimeError("AIPerf aggregation requires between 2 and 10 exports")
+ collected: dict[tuple[str, str, str], list[float]] = {}
+ metric_units: dict[tuple[str, str], str] = {}
+ for path in exports:
+ try:
+ document = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, json.JSONDecodeError) as error:
+ raise RuntimeError(f"could not read AIPerf export {path}: {error}") from error
+ if not isinstance(document, dict):
+ raise RuntimeError(f"AIPerf export is not a JSON object: {path}")
+ version = document.get("aiperf_version")
+ if version != SUPPORTED_AIPERF_VERSION:
+ raise RuntimeError(
+ f"AIPerf export {path} has version {version!r}; expected {SUPPORTED_AIPERF_VERSION}"
+ )
+ error_count = document.get("error_request_count")
+ clean_error_summary = document.get("error_summary") == []
+ if error_count is None and ("error_request_count" in document or clean_error_summary):
+ document["error_request_count"] = {"unit": "requests", "avg": 0.0}
+ for metric_name, statistic in _REQUIRED_STATISTICS:
+ metric = document.get(metric_name)
+ value = metric.get(statistic) if isinstance(metric, dict) else None
+ unit = metric.get("unit") if isinstance(metric, dict) else None
+ if (
+ not isinstance(value, int | float)
+ or isinstance(value, bool)
+ or not isinstance(unit, str)
+ ):
+ raise RuntimeError(f"AIPerf export {path} has no numeric {metric_name}.{statistic}")
+ for metric_name, metric in document.items():
+ if not isinstance(metric, dict) or not isinstance(metric.get("unit"), str):
+ continue
+ unit = metric["unit"]
+ for statistic, value in metric.items():
+ if statistic == "unit" or isinstance(value, bool):
+ continue
+ if isinstance(value, int | float):
+ previous_unit = metric_units.setdefault((metric_name, statistic), unit)
+ if unit != previous_unit:
+ raise RuntimeError(
+ "AIPerf exports disagree on the unit for "
+ f"{metric_name}.{statistic}: {previous_unit!r} and {unit!r}"
+ )
+ collected.setdefault((metric_name, statistic, unit), []).append(float(value))
+ metrics = {
+ f"{metric}_{statistic}": _summary(values, unit)
+ for (metric, statistic, unit), values in collected.items()
+ if len(values) == len(exports)
+ }
+ output_path.parent.mkdir(parents=True, exist_ok=False)
+ output_path.write_text(
+ f"{json.dumps({'aiperf_version': SUPPORTED_AIPERF_VERSION, 'metrics': metrics}, indent=2)}\n",
+ encoding="utf-8",
+ )
+ return output_path
diff --git a/scripts/benchmark_routing_algorithms.py b/scripts/benchmark_routing_algorithms.py
new file mode 100755
index 000000000..9a0a8baa7
--- /dev/null
+++ b/scripts/benchmark_routing_algorithms.py
@@ -0,0 +1,1523 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Compare Switchyard routes with Rust-owned scenarios, oha, and NVIDIA AIPerf."""
+
+import argparse
+import csv
+import json
+import math
+import os
+import shutil
+import subprocess
+import sys
+import urllib.error
+import urllib.request
+from collections.abc import Sequence
+from dataclasses import asdict, dataclass, replace
+from datetime import datetime, timezone
+from pathlib import Path
+
+if __package__:
+ from .aiperf_runner import aggregate_exports, run_profile, validate_aiperf_version
+ from .routing_overhead_plot import OverheadPlotRow, write_overhead_plot
+else:
+ from aiperf_runner import aggregate_exports, run_profile, validate_aiperf_version
+ from routing_overhead_plot import OverheadPlotRow, write_overhead_plot
+
+MAX_MATERIALIZED_REQUESTS = 1_000_000
+MAX_MATERIALIZED_BYTES = 256 * 1024 * 1024
+AIPERF_LIFECYCLE_ALLOWANCE_SECONDS = 120
+
+
+@dataclass(frozen=True)
+class RequiredBinary:
+ """One command the benchmark must find before it starts."""
+
+ label: str
+ env_var: str
+ value: str
+ setup: str
+
+
+@dataclass(frozen=True)
+class DirectBaseline:
+ """A backend endpoint that AIPerf calls without going through Switchyard."""
+
+ base_url: str
+ model: str
+ api_key_env: str | None = None
+
+
+@dataclass(frozen=True)
+class BenchmarkConfig:
+ """Inputs shared by every algorithm comparison run."""
+
+ base_url: str
+ models: tuple[tuple[str, str], ...]
+ concurrency: int
+ request_count: int
+ backend_label: str
+ output_dir: Path
+ oha_bin: str
+ aiperf_bin: str
+ soak_bin: str
+ tokenizer: str = "builtin"
+ scenario_set: str = "standard"
+ scenarios: tuple[str, ...] = ()
+ load_profiles: tuple[str, ...] = ("fixed",)
+ profile_runs: int = 3
+ prompt_bytes: int = 1024
+ max_output_tokens: int = 32
+ context_window_tokens: int = 32_768
+ request_rate: float | None = None
+ scenario_backend_reset_url: str | None = None
+ direct_baseline: DirectBaseline | None = None
+
+
+@dataclass(frozen=True)
+class BenchmarkArm:
+ """One endpoint and model pair included in the comparison."""
+
+ label: str
+ model: str
+ base_url: str
+ bypasses_switchyard: bool
+ api_key_env: str | None = None
+
+
+@dataclass(frozen=True)
+class ScenarioDefinition:
+ """One validated entry exported by switchyard-soak."""
+
+ id: str
+ group: str
+ description: str
+ expected: str
+ expected_error_rate_min: float
+ expected_error_rate_max: float
+ input_file: Path
+ load_profiles: tuple[dict[str, object], ...]
+
+
+@dataclass(frozen=True)
+class RoutingDelta:
+ """Routing counters attributable to one sequential AIPerf run."""
+
+ model_calls: str
+ model_share: str
+ model_errors: str
+ classifier_calls: int
+ classifier_errors: int
+ classifier_latency_avg_ms: float | None
+ routing_overhead_avg_ms: float | None
+
+
+@dataclass(frozen=True)
+class BenchmarkResult:
+ """Selected load-tool and routing metrics for one comparison row."""
+
+ algorithm: str
+ model: str
+ scenario: str
+ scenario_description: str
+ scenario_group: str
+ load_profile: str
+ expected_behavior: str
+ expected_error_rate_min: float
+ expected_error_rate_max: float
+ expectation_met: bool
+ oha_requests_per_second: float | None
+ oha_latency_p50_ms: float | None
+ oha_latency_p99_ms: float | None
+ aiperf_requests_per_second: float | None
+ aiperf_request_latency_p50_ms: float | None
+ aiperf_ttft_p50_ms: float | None
+ aiperf_ttft_p99_ms: float | None
+ aiperf_itl_p50_ms: float | None
+ aiperf_output_tokens_per_second: float | None
+ aiperf_output_tokens_per_second_per_user_p50: float | None
+ aiperf_error_requests: float
+ aiperf_error_rate: float
+ aiperf_request_throughput_cv: float | None
+ aiperf_request_throughput_ci_low: float | None
+ aiperf_request_throughput_ci_high: float | None
+ selected_model_calls: str
+ selected_model_share: str
+ selected_model_errors: str
+ classifier_calls: int
+ classifier_errors: int
+ classifier_latency_avg_ms: float | None
+ routing_overhead_avg_ms: float | None
+ bypasses_switchyard: bool
+ aiperf_request_throughput_delta_pct: float | None = None
+ aiperf_request_latency_p50_delta_ms: float | None = None
+ aiperf_ttft_p50_delta_ms: float | None = None
+ aiperf_ttft_p50_delta_pct: float | None = None
+ aiperf_ttft_p99_delta_ms: float | None = None
+ aiperf_output_tokens_per_second_delta: float | None = None
+ aiperf_output_tokens_per_second_delta_pct: float | None = None
+
+
+def positive_int(value: str) -> int:
+ """Parse a command-line integer that must be greater than zero."""
+ parsed = int(value)
+ if parsed <= 0:
+ raise argparse.ArgumentTypeError("must be greater than zero")
+ return parsed
+
+
+def positive_float(value: str) -> float:
+ """Parse a finite command-line number that must be greater than zero."""
+ parsed = float(value)
+ if parsed <= 0 or not math.isfinite(parsed):
+ raise argparse.ArgumentTypeError("must be a finite number greater than zero")
+ return parsed
+
+
+def model_spec(value: str) -> tuple[str, str]:
+ """Parse LABEL=MODEL without coupling the report label to a filesystem name."""
+ label, separator, model = value.partition("=")
+ if not separator or not label.strip() or not model.strip():
+ raise argparse.ArgumentTypeError("must use LABEL=MODEL")
+ return label.strip(), model.strip()
+
+
+def parser() -> argparse.ArgumentParser:
+ """Build the command-line interface."""
+ command = argparse.ArgumentParser(
+ description=(
+ "Export Rust-owned request scenarios, run oha for the short baseline and AIPerf "
+ "for every selected scenario, then write one routing performance report."
+ ),
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ )
+ command.add_argument("--base-url", required=True, help="URL of a running Switchyard server")
+ command.add_argument(
+ "--direct-base-url",
+ help="backend URL for an AIPerf baseline that bypasses Switchyard",
+ )
+ command.add_argument(
+ "--direct-model",
+ help="backend model id for the direct baseline; requires --direct-base-url",
+ )
+ command.add_argument(
+ "--direct-api-key-env",
+ help="environment variable containing the direct backend API key",
+ )
+ command.add_argument(
+ "--model",
+ action="append",
+ required=True,
+ type=model_spec,
+ dest="models",
+ metavar="LABEL=MODEL",
+ help="report label and exact model id; repeat for every route to compare",
+ )
+ command.add_argument("--concurrency", type=positive_int, default=4)
+ command.add_argument("--request-count", type=positive_int, default=100)
+ command.add_argument("--tokenizer", default="builtin")
+ command.add_argument(
+ "--scenario-set",
+ choices=("core", "agentic", "resilience", "standard", "all"),
+ default="standard",
+ )
+ command.add_argument(
+ "--scenario",
+ action="append",
+ default=[],
+ help="exact Rust scenario id; repeat to preserve an explicit order",
+ )
+ command.add_argument(
+ "--load-profile",
+ action="append",
+ choices=("fixed", "concurrency-knee", "traffic-burst", "all"),
+ default=[],
+ help="load schedule to run; repeat as needed",
+ )
+ command.add_argument(
+ "--profile-runs",
+ type=positive_int,
+ default=3,
+ help="AIPerf repetitions used for confidence intervals; maximum 10",
+ )
+ command.add_argument(
+ "--request-rate",
+ type=positive_float,
+ help="base requests/second for traffic-burst; defaults to concurrency",
+ )
+ command.add_argument("--prompt-bytes", type=positive_int, default=1024)
+ command.add_argument("--max-output-tokens", type=positive_int, default=32)
+ command.add_argument("--context-window-tokens", type=positive_int, default=32_768)
+ command.add_argument("--backend-label", default="unspecified backend")
+ command.add_argument(
+ "--scenario-backend-reset-url",
+ help="optional local scenario-backend endpoint reset before each AIPerf cell",
+ )
+ command.add_argument("--output-dir", type=Path)
+ command.add_argument("--oha-bin", default=os.environ.get("OHA_BIN", "oha"))
+ command.add_argument("--aiperf-bin", default=os.environ.get("AIPERF_BIN", "aiperf"))
+ command.add_argument(
+ "--soak-bin",
+ default=os.environ.get("SWITCHYARD_SOAK_BIN", "target/release/switchyard-soak"),
+ )
+ return command
+
+
+def find_binary(value: str) -> str | None:
+ """Return an executable path, or None when the command cannot run."""
+ if os.sep in value:
+ path = Path(value).expanduser().resolve()
+ if path.is_file() and os.access(path, os.X_OK):
+ return str(path)
+ else:
+ found = shutil.which(value)
+ if found is not None:
+ return found
+ return None
+
+
+def resolve_binaries(requirements: Sequence[RequiredBinary]) -> dict[str, str]:
+ """Resolve every command and print all missing-command setup instructions."""
+ resolved = {}
+ missing = []
+ for requirement in requirements:
+ path = find_binary(requirement.value)
+ if path is None:
+ missing.append(requirement)
+ else:
+ resolved[requirement.label] = path
+ for requirement in missing:
+ print(
+ f"warning: {requirement.label} executable not found: {requirement.value}",
+ file=sys.stderr,
+ )
+ print(f" {requirement.setup}", file=sys.stderr)
+ print(
+ f" Or set {requirement.env_var}=/path/to/{Path(requirement.value).name}",
+ file=sys.stderr,
+ )
+ if missing:
+ raise RuntimeError(f"install or build the {len(missing)} missing command(s), then rerun")
+ return resolved
+
+
+def run_checked(name: str, command: Sequence[str], log_path: Path) -> None:
+ """Run one finite tool and keep its output in a named log file."""
+ print(f"Running {name}; log: {log_path}")
+ with log_path.open("w", encoding="utf-8") as log:
+ result = subprocess.run(
+ list(command),
+ stdout=log,
+ stderr=subprocess.STDOUT,
+ text=True,
+ check=False,
+ )
+ if result.returncode != 0:
+ raise RuntimeError(f"{name} failed with status {result.returncode}; see {log_path}")
+
+
+def read_json_object(path: Path) -> dict[str, object]:
+ """Read a JSON object and name the invalid result file in any error."""
+ try:
+ document = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, json.JSONDecodeError) as error:
+ raise RuntimeError(f"could not read JSON result {path}: {error}") from error
+ if not isinstance(document, dict):
+ raise RuntimeError(f"expected a JSON object in {path}")
+ return document
+
+
+def capture_stats(base_url: str, path: Path) -> dict[str, object]:
+ """Capture Switchyard's routing counters before or after one isolated run."""
+ try:
+ with urllib.request.urlopen(f"{base_url.rstrip('/')}/v1/stats", timeout=10) as response:
+ document = json.loads(response.read())
+ except (OSError, urllib.error.URLError, json.JSONDecodeError) as error:
+ raise RuntimeError(f"could not read Switchyard stats: {error}") from error
+ if not isinstance(document, dict):
+ raise RuntimeError("Switchyard /v1/stats did not return a JSON object")
+ path.write_text(f"{json.dumps(document, indent=2)}\n", encoding="utf-8")
+ return document
+
+
+def reset_scenario_backend(url: str | None) -> None:
+ """Reset local failure injection so every algorithm receives the same attempts."""
+ if url is None:
+ return
+ request = urllib.request.Request(url, data=b"", method="POST")
+ try:
+ with urllib.request.urlopen(request, timeout=10) as response:
+ document = json.loads(response.read())
+ except (OSError, urllib.error.URLError, json.JSONDecodeError) as error:
+ raise RuntimeError(f"could not reset the scenario backend: {error}") from error
+ status = document.get("status") if isinstance(document, dict) else None
+ if response.status != 200 or status != "reset":
+ raise RuntimeError(f"scenario backend reset returned an invalid response from {url}")
+
+
+def _nested_number(document: dict[str, object], section: str, field: str) -> float:
+ section_value = document.get(section)
+ value = section_value.get(field) if isinstance(section_value, dict) else None
+ return float(value) if isinstance(value, int | float) else 0.0
+
+
+def _histogram_delta(
+ before: dict[str, object], after: dict[str, object], histogram: str
+) -> float | None:
+ count = _nested_number(after, histogram, "count") - _nested_number(before, histogram, "count")
+ total = _nested_number(after, histogram, "total_ms") - _nested_number(
+ before, histogram, "total_ms"
+ )
+ return total / count if count > 0 else None
+
+
+def stats_delta(before: dict[str, object], after: dict[str, object]) -> RoutingDelta:
+ """Return routing and target-counter changes for one run."""
+ before_value = before.get("models")
+ after_value = after.get("models")
+ before_models: dict[str, object] = before_value if isinstance(before_value, dict) else {}
+ after_models: dict[str, object] = after_value if isinstance(after_value, dict) else {}
+ classifier_before = before.get("classifier")
+ classifier_after = after.get("classifier")
+ before_classifier_models = (
+ classifier_before.get("models") if isinstance(classifier_before, dict) else None
+ )
+ after_classifier_models = (
+ classifier_after.get("models") if isinstance(classifier_after, dict) else None
+ )
+ calls = {}
+ errors = {}
+ for model, current in after_models.items():
+ if not isinstance(model, str) or not isinstance(current, dict):
+ continue
+ previous = before_models.get(model)
+ previous_calls = previous.get("calls", 0) if isinstance(previous, dict) else 0
+ classifier_current = (
+ after_classifier_models.get(model)
+ if isinstance(after_classifier_models, dict)
+ else None
+ )
+ classifier_previous = (
+ before_classifier_models.get(model)
+ if isinstance(before_classifier_models, dict)
+ else None
+ )
+ classifier_call_delta = float(
+ classifier_current.get("calls", 0) if isinstance(classifier_current, dict) else 0
+ ) - float(
+ classifier_previous.get("calls", 0) if isinstance(classifier_previous, dict) else 0
+ )
+ call_delta = int(
+ float(current.get("calls", 0)) - float(previous_calls) - classifier_call_delta
+ )
+ if call_delta > 0:
+ calls[model] = call_delta
+ previous_errors = previous.get("errors", 0) if isinstance(previous, dict) else 0
+ classifier_error_delta = float(
+ classifier_current.get("errors", 0) if isinstance(classifier_current, dict) else 0
+ ) - float(
+ classifier_previous.get("errors", 0) if isinstance(classifier_previous, dict) else 0
+ )
+ error_delta = int(
+ float(current.get("errors", 0)) - float(previous_errors) - classifier_error_delta
+ )
+ if error_delta > 0:
+ errors[model] = error_delta
+
+ classifier_count = classifier_total = 0.0
+ if isinstance(after_classifier_models, dict):
+ for model, current in after_classifier_models.items():
+ if not isinstance(model, str) or not isinstance(current, dict):
+ continue
+ previous = (
+ before_classifier_models.get(model)
+ if isinstance(before_classifier_models, dict)
+ else None
+ )
+ before_record = previous if isinstance(previous, dict) else {}
+ classifier_count += _nested_number(current, "model_call_latency", "count")
+ classifier_count -= _nested_number(before_record, "model_call_latency", "count")
+ classifier_total += _nested_number(current, "model_call_latency", "total_ms")
+ classifier_total -= _nested_number(before_record, "model_call_latency", "total_ms")
+
+ total_calls = sum(calls.values())
+ shares = {model: round(call_count / total_calls, 4) for model, call_count in calls.items()}
+ return RoutingDelta(
+ model_calls=json.dumps(calls, sort_keys=True, separators=(",", ":")),
+ model_share=json.dumps(shares, sort_keys=True, separators=(",", ":")),
+ model_errors=json.dumps(errors, sort_keys=True, separators=(",", ":")),
+ classifier_calls=int(
+ _nested_number(after, "classifier", "total_requests")
+ - _nested_number(before, "classifier", "total_requests")
+ ),
+ classifier_errors=int(
+ _nested_number(after, "classifier", "total_errors")
+ - _nested_number(before, "classifier", "total_errors")
+ ),
+ classifier_latency_avg_ms=(
+ classifier_total / classifier_count if classifier_count > 0 else None
+ ),
+ routing_overhead_avg_ms=_histogram_delta(before, after, "routing_overhead"),
+ )
+
+
+def validate_load_profile(profile: dict[str, object], scenario_id: str) -> None:
+ """Reject malformed or unbounded schedules before invoking a load generator."""
+ profile_id = profile.get("id")
+ kind = profile.get("kind")
+ if (profile_id, kind) == ("fixed", "fixed"):
+ return
+ if (profile_id, kind) == ("concurrency-knee", "concurrency_knee"):
+ steps = profile.get("concurrency_steps")
+ if (
+ not isinstance(steps, list)
+ or not steps
+ or not all(isinstance(step, int) and 1 <= step <= 10_000 for step in steps)
+ ):
+ raise RuntimeError(f"invalid concurrency knee for scenario {scenario_id}")
+ return
+ if (profile_id, kind) == ("traffic-burst", "traffic_burst"):
+ duration = profile.get("duration_seconds")
+ points = profile.get("points")
+ if not isinstance(duration, int) or not 1 <= duration <= 3_600:
+ raise RuntimeError(f"invalid traffic-burst duration for scenario {scenario_id}")
+ if not isinstance(points, list) or not 2 <= len(points) <= 64:
+ raise RuntimeError(f"invalid traffic-burst points for scenario {scenario_id}")
+ previous = -1.0
+ for point in points:
+ time_s = point.get("time_s") if isinstance(point, dict) else None
+ multiplier = point.get("rate_multiplier") if isinstance(point, dict) else None
+ if (
+ not isinstance(time_s, int | float)
+ or not isinstance(multiplier, int | float)
+ or not math.isfinite(float(time_s))
+ or not math.isfinite(float(multiplier))
+ or float(time_s) <= previous
+ or float(time_s) > duration
+ or float(multiplier) <= 0
+ ):
+ raise RuntimeError(f"invalid traffic-burst point for scenario {scenario_id}")
+ previous = float(time_s)
+ return
+ raise RuntimeError(f"unknown load profile for scenario {scenario_id}: {profile_id!r}")
+
+
+def export_scenarios(config: BenchmarkConfig, index: int, model: str) -> list[ScenarioDefinition]:
+ """Ask the Rust crate for the only authoritative scenario manifest."""
+ output_dir = config.output_dir / "scenario-definitions" / f"algorithm-{index:02d}"
+ command = [
+ config.soak_bin,
+ "--model",
+ model,
+ "--scenario-set",
+ config.scenario_set,
+ "--prompt-bytes",
+ str(config.prompt_bytes),
+ "--max-output-tokens",
+ str(config.max_output_tokens),
+ "--context-window-tokens",
+ str(config.context_window_tokens),
+ "--export-scenarios",
+ str(output_dir),
+ ]
+ for scenario in config.scenarios:
+ command.extend(("--scenario", scenario))
+ run_checked(
+ "scenario export",
+ command,
+ config.output_dir / f"scenario-export-{index:02d}.log",
+ )
+ document = read_json_object(output_dir / "manifest.json")
+ if document.get("schema_version") != 1 or document.get("model") != model:
+ raise RuntimeError(f"unsupported scenario manifest in {output_dir}")
+ raw_scenarios = document.get("scenarios")
+ if not isinstance(raw_scenarios, list) or not raw_scenarios:
+ raise RuntimeError(f"scenario manifest has no scenarios: {output_dir}")
+ definitions = []
+ seen = set()
+ root = output_dir.resolve()
+ for entry in raw_scenarios:
+ if not isinstance(entry, dict):
+ raise RuntimeError(f"invalid scenario entry in {output_dir}")
+ scenario_id = entry.get("id")
+ group = entry.get("group")
+ description = entry.get("description")
+ expected = entry.get("expected")
+ expected_error_rate = entry.get("expected_error_rate")
+ min_error_rate = (
+ expected_error_rate.get("min_rate") if isinstance(expected_error_rate, dict) else None
+ )
+ max_error_rate = (
+ expected_error_rate.get("max_rate") if isinstance(expected_error_rate, dict) else None
+ )
+ input_name = entry.get("input_file")
+ profiles = entry.get("load_profiles")
+ if (
+ not isinstance(scenario_id, str)
+ or not scenario_id
+ or scenario_id in seen
+ or group not in {"core", "agentic", "resilience"}
+ or not isinstance(description, str)
+ or not description
+ or not isinstance(expected, str)
+ or not expected
+ or not isinstance(min_error_rate, int | float)
+ or not isinstance(max_error_rate, int | float)
+ or not 0 <= float(min_error_rate) <= float(max_error_rate) <= 1
+ or not isinstance(input_name, str)
+ or not isinstance(profiles, list)
+ ):
+ raise RuntimeError(f"invalid scenario entry in {output_dir}")
+ input_file = (output_dir / input_name).resolve()
+ if not input_file.is_relative_to(root) or not input_file.is_file():
+ raise RuntimeError(f"scenario input escapes or is missing from {output_dir}")
+ if not all(
+ isinstance(profile, dict) and isinstance(profile.get("id"), str) for profile in profiles
+ ):
+ raise RuntimeError(f"invalid load profile for scenario {scenario_id}")
+ for profile in profiles:
+ validate_load_profile(profile, scenario_id)
+ seen.add(scenario_id)
+ definitions.append(
+ ScenarioDefinition(
+ id=scenario_id,
+ group=group,
+ description=description,
+ expected=expected,
+ expected_error_rate_min=float(min_error_rate),
+ expected_error_rate_max=float(max_error_rate),
+ input_file=input_file,
+ load_profiles=tuple(profiles),
+ )
+ )
+ return definitions
+
+
+def selected_profiles(
+ config: BenchmarkConfig, scenario: ScenarioDefinition
+) -> list[dict[str, object]]:
+ """Select only schedules supported by this request shape."""
+ requested = set(config.load_profiles)
+ if "all" in requested:
+ return list(scenario.load_profiles)
+ return [profile for profile in scenario.load_profiles if profile.get("id") in requested]
+
+
+def _metric(document: dict[str, object], name: str, statistic: str) -> tuple[float | None, object]:
+ group = document.get(name)
+ if isinstance(group, dict):
+ value = group.get(statistic)
+ return (float(value) if isinstance(value, int | float) else None, group.get("unit"))
+ metrics = document.get("metrics")
+ aggregate = metrics.get(f"{name}_{statistic}") if isinstance(metrics, dict) else None
+ if isinstance(aggregate, dict):
+ value = aggregate.get("mean")
+ return (
+ float(value) if isinstance(value, int | float) else None,
+ aggregate.get("unit"),
+ )
+ return None, None
+
+
+def _latency(document: dict[str, object], metric: str, statistic: str, path: Path) -> float | None:
+ value, unit = _metric(document, metric, statistic)
+ if value is None:
+ return None
+ if unit == "ms":
+ return value
+ if unit in {"s", "seconds"}:
+ return value * 1000
+ raise RuntimeError(f"unsupported {metric} unit {unit!r} in {path}")
+
+
+def parse_result(
+ arm: BenchmarkArm,
+ scenario: ScenarioDefinition,
+ load_profile: str,
+ oha_path: Path | None,
+ aiperf_path: Path,
+ before_stats: dict[str, object],
+ after_stats: dict[str, object],
+) -> BenchmarkResult:
+ """Combine one AIPerf run, an optional oha baseline, and routing counter deltas."""
+ oha_rps = oha_p50 = oha_p99 = None
+ if oha_path is not None:
+ oha = read_json_object(oha_path)
+ success, _unit = _metric(oha, "summary", "successRate")
+ if success != 1:
+ raise RuntimeError(f"oha success rate for {arm.model} was {success}; see {oha_path}")
+ oha_rps, _unit = _metric(oha, "summary", "requestsPerSec")
+ p50, _unit = _metric(oha, "latencyPercentiles", "p50")
+ p99, _unit = _metric(oha, "latencyPercentiles", "p99")
+ oha_p50 = p50 * 1000 if p50 is not None else None
+ oha_p99 = p99 * 1000 if p99 is not None else None
+ aiperf = read_json_object(aiperf_path)
+ errors, _unit = _metric(aiperf, "error_request_count", "avg")
+ error_count = errors or 0.0
+ measured_requests, _unit = _metric(aiperf, "request_count", "avg")
+ error_rate = (
+ error_count / measured_requests
+ if measured_requests is not None and measured_requests > 0
+ else 1.0
+ )
+ expectation_met = (
+ scenario.expected_error_rate_min <= error_rate <= scenario.expected_error_rate_max
+ )
+ throughput, _unit = _metric(aiperf, "request_throughput", "avg")
+ output_throughput, _unit = _metric(aiperf, "output_token_throughput", "avg")
+ user_throughput, _unit = _metric(aiperf, "e2e_output_token_throughput", "p50")
+ aggregate = aiperf.get("metrics")
+ throughput_stats = (
+ aggregate.get("request_throughput_avg") if isinstance(aggregate, dict) else None
+ )
+ routing = stats_delta(before_stats, after_stats)
+ return BenchmarkResult(
+ algorithm=arm.label,
+ model=arm.model,
+ scenario=scenario.id,
+ scenario_description=scenario.description,
+ scenario_group=scenario.group,
+ load_profile=load_profile,
+ expected_behavior=scenario.expected,
+ expected_error_rate_min=scenario.expected_error_rate_min,
+ expected_error_rate_max=scenario.expected_error_rate_max,
+ expectation_met=expectation_met,
+ oha_requests_per_second=oha_rps,
+ oha_latency_p50_ms=oha_p50,
+ oha_latency_p99_ms=oha_p99,
+ aiperf_requests_per_second=throughput,
+ aiperf_request_latency_p50_ms=_latency(aiperf, "request_latency", "p50", aiperf_path),
+ aiperf_ttft_p50_ms=_latency(aiperf, "time_to_first_token", "p50", aiperf_path),
+ aiperf_ttft_p99_ms=_latency(aiperf, "time_to_first_token", "p99", aiperf_path),
+ aiperf_itl_p50_ms=_latency(aiperf, "inter_token_latency", "p50", aiperf_path),
+ aiperf_output_tokens_per_second=output_throughput,
+ aiperf_output_tokens_per_second_per_user_p50=user_throughput,
+ aiperf_error_requests=error_count,
+ aiperf_error_rate=error_rate,
+ aiperf_request_throughput_cv=(
+ float(throughput_stats["cv"])
+ if isinstance(throughput_stats, dict)
+ and isinstance(throughput_stats.get("cv"), int | float)
+ else None
+ ),
+ aiperf_request_throughput_ci_low=(
+ float(throughput_stats["ci_low"])
+ if isinstance(throughput_stats, dict)
+ and isinstance(throughput_stats.get("ci_low"), int | float)
+ else None
+ ),
+ aiperf_request_throughput_ci_high=(
+ float(throughput_stats["ci_high"])
+ if isinstance(throughput_stats, dict)
+ and isinstance(throughput_stats.get("ci_high"), int | float)
+ else None
+ ),
+ selected_model_calls=routing.model_calls,
+ selected_model_share=routing.model_share,
+ selected_model_errors=routing.model_errors,
+ classifier_calls=routing.classifier_calls,
+ classifier_errors=routing.classifier_errors,
+ classifier_latency_avg_ms=routing.classifier_latency_avg_ms,
+ routing_overhead_avg_ms=routing.routing_overhead_avg_ms,
+ bypasses_switchyard=arm.bypasses_switchyard,
+ )
+
+
+def _percent_change(value: float | None, baseline: float | None) -> float | None:
+ if value is None or baseline is None or baseline == 0:
+ return None
+ return (value / baseline - 1) * 100
+
+
+def _difference(value: float | None, baseline: float | None) -> float | None:
+ if value is None or baseline is None:
+ return None
+ return value - baseline
+
+
+def _direct_baselines(
+ results: Sequence[BenchmarkResult],
+) -> dict[tuple[str, str], BenchmarkResult]:
+ """Index the one direct-backend result allowed for each workload."""
+ baselines = {}
+ for result in results:
+ if not result.bypasses_switchyard:
+ continue
+ key = (result.scenario, result.load_profile)
+ if key in baselines:
+ raise RuntimeError(
+ f"duplicate direct baseline for {result.scenario} {result.load_profile}"
+ )
+ baselines[key] = result
+ return baselines
+
+
+def compare_to_direct_backend(results: Sequence[BenchmarkResult]) -> list[BenchmarkResult]:
+ """Attach AIPerf deltas from the direct-backend row for each workload."""
+ baselines = _direct_baselines(results)
+ if not baselines:
+ return list(results)
+
+ compared = []
+ for result in results:
+ if result.bypasses_switchyard:
+ compared.append(result)
+ continue
+ key = (result.scenario, result.load_profile)
+ baseline = baselines.get(key)
+ if baseline is None:
+ if result.scenario_group != "resilience":
+ raise RuntimeError(
+ f"missing direct baseline for {result.scenario} {result.load_profile}"
+ )
+ compared.append(result)
+ continue
+ if result.aiperf_error_rate != 0 or baseline.aiperf_error_rate != 0:
+ compared.append(result)
+ continue
+ compared.append(
+ replace(
+ result,
+ aiperf_request_throughput_delta_pct=_percent_change(
+ result.aiperf_requests_per_second,
+ baseline.aiperf_requests_per_second,
+ ),
+ aiperf_request_latency_p50_delta_ms=_difference(
+ result.aiperf_request_latency_p50_ms,
+ baseline.aiperf_request_latency_p50_ms,
+ ),
+ aiperf_ttft_p50_delta_ms=_difference(
+ result.aiperf_ttft_p50_ms,
+ baseline.aiperf_ttft_p50_ms,
+ ),
+ aiperf_ttft_p50_delta_pct=_percent_change(
+ result.aiperf_ttft_p50_ms,
+ baseline.aiperf_ttft_p50_ms,
+ ),
+ aiperf_ttft_p99_delta_ms=_difference(
+ result.aiperf_ttft_p99_ms,
+ baseline.aiperf_ttft_p99_ms,
+ ),
+ aiperf_output_tokens_per_second_delta=_difference(
+ result.aiperf_output_tokens_per_second,
+ baseline.aiperf_output_tokens_per_second,
+ ),
+ aiperf_output_tokens_per_second_delta_pct=_percent_change(
+ result.aiperf_output_tokens_per_second,
+ baseline.aiperf_output_tokens_per_second,
+ ),
+ )
+ )
+ return compared
+
+
+def format_metric(value: float | None) -> str:
+ """Render a report value without implying precision the benchmark lacks."""
+ return "n/a" if value is None else f"{value:.2f}"
+
+
+def format_delta(value: float | None, suffix: str = "") -> str:
+ """Render a signed direct-backend comparison."""
+ return "n/a" if value is None else f"{value:+,.2f}{suffix}"
+
+
+def write_report(config: BenchmarkConfig, results: Sequence[BenchmarkResult]) -> None:
+ """Write machine-readable and reviewer-readable comparison reports."""
+ if not results:
+ raise RuntimeError("no scenario and load-profile combinations were selected")
+ rows = [asdict(result) for result in results]
+ payload = {
+ "generated_at": datetime.now(timezone.utc).isoformat(),
+ "backend": config.backend_label,
+ "base_url": config.base_url,
+ "direct_baseline": (
+ asdict(config.direct_baseline) if config.direct_baseline is not None else None
+ ),
+ "concurrency": config.concurrency,
+ "request_count": config.request_count,
+ "profile_runs": config.profile_runs,
+ "scenario_set": config.scenario_set,
+ "selected_scenarios": list(config.scenarios),
+ "selected_load_profiles": list(config.load_profiles),
+ "aiperf_tokenizer": config.tokenizer,
+ "results": rows,
+ }
+ (config.output_dir / "report.json").write_text(
+ f"{json.dumps(payload, indent=2)}\n", encoding="utf-8"
+ )
+ with (config.output_dir / "report.csv").open("w", encoding="utf-8", newline="") as output:
+ writer = csv.DictWriter(output, fieldnames=list(rows[0]))
+ writer.writeheader()
+ writer.writerows(rows)
+
+ lines = ["# Routing algorithm performance", ""]
+ run_details = [
+ f"- Backend: {config.backend_label}",
+ f"- AIPerf repetitions per row: {config.profile_runs}",
+ f"- Scenario set: {config.scenario_set}",
+ "- Run order: algorithms ran back-to-back within each scenario and load; all jobs "
+ "remained sequential",
+ ]
+ if config.direct_baseline is not None:
+ run_details.append(
+ f"- Direct baseline: `{config.direct_baseline.model}` at "
+ f"`{config.direct_baseline.base_url}`; this arm bypasses Switchyard"
+ )
+ overhead_results = [
+ result
+ for result in results
+ if not result.bypasses_switchyard and result.scenario_group != "resilience"
+ ]
+ if overhead_results:
+ write_overhead_plot(
+ config.output_dir / "routing-overhead.svg",
+ [
+ OverheadPlotRow(
+ scenario=result.scenario,
+ load=result.load_profile,
+ route=result.algorithm,
+ ttft_delta_ms=result.aiperf_ttft_p50_delta_ms,
+ ttft_delta_pct=result.aiperf_ttft_p50_delta_pct,
+ token_throughput_delta=(result.aiperf_output_tokens_per_second_delta),
+ token_throughput_delta_pct=(
+ result.aiperf_output_tokens_per_second_delta_pct
+ ),
+ )
+ for result in overhead_results
+ ],
+ )
+ lines.extend(
+ (
+ "## Routing overhead versus the direct backend",
+ "",
+ "The direct arm sends the same Rust-exported requests without Switchyard. These "
+ "deltas isolate Switchyard overhead only when both arms use the same backend "
+ "deployment, model, and response settings. Positive latency means the routed "
+ "request took longer. Negative throughput means the routed path processed less "
+ "work. Resilience workloads are shown later but are not compared because their "
+ "failure behavior differs.",
+ "",
+ )
+ )
+ if overhead_results:
+ lines.extend(
+ (
+ "![Routing overhead plots](routing-overhead.svg)",
+ "",
+ )
+ )
+ if not overhead_results:
+ lines.append("No successful direct-backend comparisons were available.")
+ else:
+ lines.extend(
+ (
+ "| Workload | Route | Load | Request p50 change (ms) | "
+ "TTFT p50 change (ms) | TTFT change (%) | Request throughput change (%) | "
+ "Token throughput change (tokens/s) | Token throughput change (%) |",
+ "|---|---|---|---:|---:|---:|---:|---:|---:|",
+ )
+ )
+ for result in overhead_results:
+ lines.append(
+ f"| {result.scenario.replace('-', ' ')} | {result.algorithm} | "
+ f"{result.load_profile} | "
+ f"{format_delta(result.aiperf_request_latency_p50_delta_ms, ' ms')} | "
+ f"{format_delta(result.aiperf_ttft_p50_delta_ms, ' ms')} | "
+ f"{format_delta(result.aiperf_ttft_p50_delta_pct, '%')} | "
+ f"{format_delta(result.aiperf_request_throughput_delta_pct, '%')} | "
+ f"{format_delta(result.aiperf_output_tokens_per_second_delta, ' tokens/s')} | "
+ f"{format_delta(result.aiperf_output_tokens_per_second_delta_pct, '%')} |"
+ )
+ lines.extend(
+ (
+ "",
+ "Use the confidence intervals in the detailed results before treating a small "
+ "throughput change as meaningful. A small negative latency or positive "
+ "throughput delta is run-to-run noise when those intervals overlap. Request and "
+ "token throughput percentage changes match when every arm emits the same fixed "
+ "output length; the detailed table shows the absolute token rates.",
+ "",
+ "## Workloads measured",
+ "",
+ "| Workload | Request pattern |",
+ "|---|---|",
+ )
+ )
+ seen_scenarios = set()
+ for result in overhead_results:
+ if result.scenario in seen_scenarios:
+ continue
+ seen_scenarios.add(result.scenario)
+ lines.append(f"| {result.scenario.replace('-', ' ')} | {result.scenario_description} |")
+ lines.extend(("", "## Run details", "", *run_details))
+ lines.extend(
+ (
+ "",
+ "## Throughput and latency",
+ "",
+ "| Algorithm | Scenario | Load | oha req/s | AIPerf req/s | request p50 ms | "
+ "TTFT p50 ms | TTFT p99 ms | ITL p50 ms | output tok/s | error rate | gate |",
+ "|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---|",
+ )
+ )
+ for result in results:
+ if result.scenario_group == "resilience":
+ continue
+ lines.append(
+ f"| {result.algorithm} | {result.scenario} | {result.load_profile} | "
+ f"{format_metric(result.oha_requests_per_second)} | "
+ f"{format_metric(result.aiperf_requests_per_second)} | "
+ f"{format_metric(result.aiperf_request_latency_p50_ms)} | "
+ f"{format_metric(result.aiperf_ttft_p50_ms)} | "
+ f"{format_metric(result.aiperf_ttft_p99_ms)} | "
+ f"{format_metric(result.aiperf_itl_p50_ms)} | "
+ f"{format_metric(result.aiperf_output_tokens_per_second)} | "
+ f"{result.aiperf_error_rate:.2%} | "
+ f"{'PASS' if result.expectation_met else 'FAIL'} |"
+ )
+ if config.profile_runs > 1:
+ lines.extend(
+ (
+ "",
+ "## Repeatability",
+ "",
+ "The script calculates variation and confidence intervals across independent "
+ "AIPerf runs. "
+ "Treat a small throughput delta as noise when these intervals overlap.",
+ "",
+ "| Algorithm | Scenario | Load | req/s average | req/s CV | req/s 95% CI |",
+ "|---|---|---|---:|---:|---:|",
+ )
+ )
+ for result in results:
+ if result.scenario_group == "resilience":
+ continue
+ confidence_interval = (
+ "n/a"
+ if result.aiperf_request_throughput_ci_low is None
+ or result.aiperf_request_throughput_ci_high is None
+ else f"{result.aiperf_request_throughput_ci_low:.2f}–"
+ f"{result.aiperf_request_throughput_ci_high:.2f}"
+ )
+ coefficient_of_variation = (
+ "n/a"
+ if result.aiperf_request_throughput_cv is None
+ else f"{result.aiperf_request_throughput_cv:.2%}"
+ )
+ lines.append(
+ f"| {result.algorithm} | {result.scenario} | {result.load_profile} | "
+ f"{format_metric(result.aiperf_requests_per_second)} | "
+ f"{coefficient_of_variation} | {confidence_interval} |"
+ )
+ lines.extend(
+ (
+ "",
+ "## Routing behavior",
+ "",
+ "| Algorithm | Scenario | Load | selected target calls | selected target share | target errors | "
+ "classifier calls | classifier errors | classifier avg ms | "
+ "Switchyard routing time avg ms |",
+ "|---|---|---|---|---|---|---:|---:|---:|---:|",
+ )
+ )
+ for result in results:
+ if result.bypasses_switchyard:
+ continue
+ lines.append(
+ f"| {result.algorithm} | {result.scenario} | {result.load_profile} | "
+ f"`{result.selected_model_calls}` | `{result.selected_model_share}` | "
+ f"`{result.selected_model_errors}` | "
+ f"{result.classifier_calls} | {result.classifier_errors} | "
+ f"{format_metric(result.classifier_latency_avg_ms)} | "
+ f"{format_metric(result.routing_overhead_avg_ms)} |"
+ )
+
+ resilience = [result for result in results if result.scenario_group == "resilience"]
+ if resilience:
+ lines.extend(
+ (
+ "",
+ "## Resilience",
+ "",
+ "| Algorithm | Scenario | Load | client error rate | expected range | gate | "
+ "target errors | Expected behavior |",
+ "|---|---|---|---:|---|---|---|---|",
+ )
+ )
+ for result in resilience:
+ if result.bypasses_switchyard:
+ continue
+ lines.append(
+ f"| {result.algorithm} | {result.scenario} | {result.load_profile} | "
+ f"{result.aiperf_error_rate:.2%} | "
+ f"{result.expected_error_rate_min:.2%}–{result.expected_error_rate_max:.2%} | "
+ f"{'PASS' if result.expectation_met else 'FAIL'} | "
+ f"`{result.selected_model_errors}` | "
+ f"{result.expected_behavior} |"
+ )
+ lines.extend(
+ (
+ "",
+ "oha runs only for the fixed short-interactive baseline. AIPerf replays the same "
+ "Rust-exported request payload for streaming, session-aware LLM metrics. Compare "
+ "algorithms within the same scenario and load row.",
+ "",
+ "The local scenario backend isolates Switchyard routing and protocol overhead. It "
+ "does not predict model capacity. Repeat the same command against routes backed by "
+ "one real deployment to include tokenization, model execution, and provider queuing.",
+ "",
+ )
+ )
+ (config.output_dir / "report.md").write_text("\n".join(lines), encoding="utf-8")
+
+
+def run_oha(
+ config: BenchmarkConfig,
+ label: str,
+ scenario: ScenarioDefinition,
+ output_dir: Path,
+) -> Path:
+ """Run oha only for the fixed short-interactive baseline."""
+ inputs = read_json_object(scenario.input_file)
+ data = inputs.get("data")
+ try:
+ payload = data[0]["payloads"][0] # type: ignore[index]
+ except (IndexError, KeyError, TypeError) as error:
+ raise RuntimeError(f"invalid AIPerf input file {scenario.input_file}") from error
+ if not isinstance(payload, dict):
+ raise RuntimeError(f"invalid AIPerf payload in {scenario.input_file}")
+ body = dict(payload)
+ body["stream"] = False
+ request_path = output_dir / f"{label}-request.json"
+ result_path = output_dir / f"{label}.json"
+ request_path.write_text(json.dumps(body), encoding="utf-8")
+ run_checked(
+ f"oha {label}",
+ [
+ config.oha_bin,
+ "-n",
+ str(config.request_count),
+ "-c",
+ str(config.concurrency),
+ "--latency-correction",
+ "--no-tui",
+ "--method",
+ "POST",
+ "-T",
+ "application/json",
+ "-D",
+ str(request_path),
+ "--output-format",
+ "json",
+ "--output",
+ str(result_path),
+ f"{config.base_url.rstrip('/')}/v1/chat/completions",
+ ],
+ output_dir / f"{label}.log",
+ )
+ return result_path
+
+
+def materialize_aiperf_input(
+ scenario: ScenarioDefinition,
+ output_path: Path,
+ minimum_request_count: int,
+ namespace: str = "load",
+) -> Path:
+ """Repeat Rust-owned sessions with unique ids so AIPerf never wraps the dataset."""
+ if minimum_request_count > MAX_MATERIALIZED_REQUESTS:
+ raise RuntimeError(
+ f"AIPerf input needs {minimum_request_count} requests; maximum is "
+ f"{MAX_MATERIALIZED_REQUESTS}"
+ )
+ document = read_json_object(scenario.input_file)
+ data = document.get("data")
+ if not isinstance(data, list) or not data:
+ raise RuntimeError(f"AIPerf input has no sessions: {scenario.input_file}")
+
+ validated = []
+ requests_per_replica = 0
+ bytes_per_replica = 0
+ for raw_session in data:
+ if not isinstance(raw_session, dict):
+ raise RuntimeError(f"invalid AIPerf session in {scenario.input_file}")
+ session_id = raw_session.get("session_id")
+ payloads = raw_session.get("payloads")
+ if not isinstance(session_id, str) or not isinstance(payloads, list) or not payloads:
+ raise RuntimeError(f"invalid AIPerf session in {scenario.input_file}")
+ validated.append((raw_session, session_id, len(payloads)))
+ requests_per_replica += len(payloads)
+ bytes_per_replica += len(json.dumps(raw_session).encode()) + 64
+ replica_count = math.ceil(minimum_request_count / requests_per_replica)
+ estimated_bytes = replica_count * bytes_per_replica
+ if estimated_bytes > MAX_MATERIALIZED_BYTES:
+ raise RuntimeError(
+ f"AIPerf input would use about {estimated_bytes} bytes; maximum is "
+ f"{MAX_MATERIALIZED_BYTES}"
+ )
+
+ sessions = []
+ request_count = 0
+ replica = 0
+ for _replica_index in range(replica_count):
+ for raw_session, session_id, payload_count in validated:
+ sessions.append(
+ {
+ **raw_session,
+ "session_id": f"{session_id}-{namespace}-{replica:06d}",
+ }
+ )
+ request_count += payload_count
+ replica += 1
+ if request_count >= minimum_request_count:
+ break
+ if request_count >= minimum_request_count:
+ break
+ output_path.write_text(json.dumps({"data": sessions}), encoding="utf-8")
+ return output_path
+
+
+def profile_request_count(config: BenchmarkConfig, profile: dict[str, object]) -> int:
+ """Return a conservative request bound for one AIPerf schedule."""
+ if profile.get("kind") != "traffic_burst":
+ return config.request_count
+ duration = profile.get("duration_seconds")
+ points = profile.get("points")
+ if not isinstance(duration, int) or not isinstance(points, list):
+ raise RuntimeError("traffic-burst profile is missing its duration or points")
+ parsed_points = []
+ for point in points:
+ if not isinstance(point, dict):
+ raise RuntimeError("traffic-burst profile has an invalid point")
+ time_s = point.get("time_s")
+ multiplier = point.get("rate_multiplier")
+ if not isinstance(time_s, int | float) or not isinstance(multiplier, int | float):
+ raise RuntimeError("traffic-burst profile has an invalid point")
+ parsed_points.append((float(time_s), float(multiplier)))
+ request_multiplier_seconds = sum(
+ (right_time - left_time) * (left_rate + right_rate) / 2
+ for (left_time, left_rate), (right_time, right_rate) in zip(
+ parsed_points, parsed_points[1:], strict=False
+ )
+ )
+ return math.ceil(request_multiplier_seconds * (config.request_rate or config.concurrency))
+
+
+def aiperf_timeout_seconds(
+ config: BenchmarkConfig,
+ scenario: ScenarioDefinition,
+ profile: dict[str, object],
+ concurrency: int | None,
+) -> int:
+ """Bound one process by its schedule and per-request timeout."""
+ request_timeout = 1 if scenario.id == "client-cancellation" else 30
+ if profile.get("kind") == "traffic_burst":
+ duration = profile.get("duration_seconds")
+ if not isinstance(duration, int):
+ raise RuntimeError(f"traffic-burst profile has no duration for {scenario.id}")
+ workload_seconds = duration + request_timeout
+ else:
+ effective_concurrency = concurrency or config.concurrency
+ workload_seconds = math.ceil(config.request_count / effective_concurrency) * request_timeout
+ return AIPERF_LIFECYCLE_ALLOWANCE_SECONDS + workload_seconds
+
+
+def run_aiperf(
+ config: BenchmarkConfig,
+ arm: BenchmarkArm,
+ scenario: ScenarioDefinition,
+ profile: dict[str, object],
+ output_dir: Path,
+ artifact_label: str,
+ concurrency: int | None = None,
+) -> Path:
+ """Run independent AIPerf repetitions using Rust-exported inputs-json datasets."""
+ artifact_root = output_dir / artifact_label
+ effective_concurrency = concurrency or config.concurrency
+ load_arguments: list[str] = []
+ kind = profile.get("kind")
+ if kind == "traffic_burst":
+ points = profile.get("points")
+ if not isinstance(points, list):
+ raise RuntimeError(f"traffic-burst profile has no points for {scenario.id}")
+ base_rate = config.request_rate or float(config.concurrency)
+ rate_points = []
+ for point in points:
+ if not isinstance(point, dict):
+ raise RuntimeError(f"invalid traffic-burst point for {scenario.id}")
+ rate_points.append(
+ {
+ "time_s": point["time_s"],
+ "qps": base_rate * float(point["rate_multiplier"]),
+ }
+ )
+ series_path = output_dir / f"{artifact_label}-request-rate.json"
+ series_path.write_text(
+ f"{json.dumps({'points': rate_points}, indent=2)}\n", encoding="utf-8"
+ )
+ load_arguments.extend(
+ ("--request-rate-series", str(series_path), "--arrival-pattern", "constant")
+ )
+ load_arguments.extend(("--benchmark-duration", str(profile.get("duration_seconds"))))
+ load_arguments.extend(("--concurrency", str(config.concurrency)))
+ else:
+ load_arguments.extend(("--concurrency", str(effective_concurrency)))
+ load_arguments.extend(("--request-count", str(config.request_count)))
+
+ exports: list[Path] = []
+ for trial in range(1, config.profile_runs + 1):
+ run_label = f"{artifact_label}-run-{trial:02d}"
+ input_path = materialize_aiperf_input(
+ scenario,
+ output_dir / f"{run_label}-inputs.json",
+ profile_request_count(config, profile),
+ namespace=f"{artifact_label}-run-{trial:02d}",
+ )
+ command = [
+ config.aiperf_bin,
+ "profile",
+ "--model",
+ arm.model,
+ "--url",
+ arm.base_url,
+ "--endpoint-type",
+ "chat",
+ "--streaming",
+ "--tokenizer",
+ config.tokenizer,
+ "--custom-dataset-type",
+ "inputs-json",
+ "--input-file",
+ str(input_path),
+ "--session-header",
+ "x-switchyard-session-id",
+ "--random-seed",
+ "42",
+ "--num-profile-runs",
+ "1",
+ "--record-processors",
+ "1",
+ "--request-timeout-seconds",
+ "1" if scenario.id == "client-cancellation" else "30",
+ "--ui",
+ "none",
+ *load_arguments,
+ ]
+ if arm.api_key_env is not None:
+ command.extend(("--api-key", f"${{{arm.api_key_env}}}"))
+ trial_root = (
+ artifact_root / "profile_runs" / f"run_{trial:04d}"
+ if config.profile_runs > 1
+ else artifact_root
+ )
+ exports.append(
+ run_profile(
+ command,
+ output_dir / f"{run_label}.log",
+ trial_root,
+ aiperf_timeout_seconds(config, scenario, profile, concurrency),
+ )
+ )
+ if len(exports) == 1:
+ return exports[0]
+ aggregate_path: Path = aggregate_exports(
+ exports,
+ artifact_root / "aggregate" / "profile_export_aiperf_aggregate.json",
+ )
+ return aggregate_path
+
+
+def run_benchmark(config: BenchmarkConfig) -> Path:
+ """Run selected scenarios for every algorithm and return the report directory."""
+ if not config.models:
+ raise RuntimeError("at least one algorithm model is required")
+ if config.profile_runs > 10:
+ raise RuntimeError("--profile-runs must be between 1 and 10")
+ validate_aiperf_version(config.aiperf_bin)
+ arms = [BenchmarkArm(label, model, config.base_url, False) for label, model in config.models]
+ if config.direct_baseline is not None:
+ arms.insert(
+ 0,
+ BenchmarkArm(
+ "direct-backend",
+ config.direct_baseline.model,
+ config.direct_baseline.base_url,
+ True,
+ config.direct_baseline.api_key_env,
+ ),
+ )
+ labels = [arm.label for arm in arms]
+ if len(labels) != len(set(labels)):
+ raise RuntimeError("algorithm labels must be unique")
+ config.output_dir.mkdir(parents=True, exist_ok=False)
+ oha_dir = config.output_dir / "oha"
+ aiperf_dir = config.output_dir / "aiperf"
+ oha_dir.mkdir()
+ aiperf_dir.mkdir()
+ results = []
+ definition_sets = [export_scenarios(config, index, arm.model) for index, arm in enumerate(arms)]
+ expected_ids = [scenario.id for scenario in definition_sets[0]]
+ for definitions in definition_sets[1:]:
+ if [scenario.id for scenario in definitions] != expected_ids:
+ raise RuntimeError("scenario exports differ between algorithms")
+
+ for scenario_index, baseline_scenario in enumerate(definition_sets[0]):
+ for profile in selected_profiles(config, baseline_scenario):
+ profile_id = str(profile["id"])
+ concurrencies: list[int | None] = [None]
+ if profile.get("kind") == "concurrency_knee":
+ steps = profile.get("concurrency_steps")
+ if not isinstance(steps, list) or not all(isinstance(step, int) for step in steps):
+ raise RuntimeError(f"invalid concurrency knee for {baseline_scenario.id}")
+ concurrencies = sorted(
+ set(
+ [step for step in steps if step <= config.concurrency]
+ + [config.concurrency]
+ )
+ )
+ for concurrency in concurrencies:
+ load_label = profile_id if concurrency is None else f"{profile_id}@{concurrency}"
+ for index, arm in enumerate(arms):
+ scenario = definition_sets[index][scenario_index]
+ if arm.bypasses_switchyard and scenario.group == "resilience":
+ continue
+ artifact_label = (
+ f"algorithm-{index:02d}-{scenario.id}-{load_label.replace('@', '-')}"
+ )
+ oha_path = None
+ if (
+ not arm.bypasses_switchyard
+ and scenario.id == "short-interactive"
+ and profile_id == "fixed"
+ ):
+ oha_path = run_oha(
+ config,
+ artifact_label,
+ scenario,
+ oha_dir,
+ )
+ reset_scenario_backend(config.scenario_backend_reset_url)
+ before = (
+ {}
+ if arm.bypasses_switchyard
+ else capture_stats(
+ config.base_url,
+ aiperf_dir / f"{artifact_label}-stats-before.json",
+ )
+ )
+ aiperf_path = run_aiperf(
+ config,
+ arm,
+ scenario,
+ profile,
+ aiperf_dir,
+ artifact_label,
+ concurrency,
+ )
+ after = (
+ {}
+ if arm.bypasses_switchyard
+ else capture_stats(
+ config.base_url,
+ aiperf_dir / f"{artifact_label}-stats-after.json",
+ )
+ )
+ results.append(
+ parse_result(
+ arm,
+ scenario,
+ load_label,
+ oha_path,
+ aiperf_path,
+ before,
+ after,
+ )
+ )
+ results = compare_to_direct_backend(results)
+ write_report(config, results)
+ failures = [result for result in results if not result.expectation_met]
+ if failures:
+ raise RuntimeError(
+ f"{len(failures)} benchmark row(s) missed their error-rate gate; "
+ f"see {config.output_dir / 'report.md'}"
+ )
+ return config.output_dir
+
+
+def default_output_dir() -> Path:
+ """Choose a new timestamped directory under the repository."""
+ repo_root = Path(__file__).resolve().parents[1]
+ stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+ return repo_root / "routing-benchmark-results" / stamp
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ """Parse arguments, run the comparison, and print one actionable failure."""
+ command = parser()
+ args = command.parse_args(argv)
+ if (args.direct_base_url is None) != (args.direct_model is None):
+ command.error("--direct-base-url and --direct-model must be used together")
+ if args.direct_api_key_env is not None and args.direct_base_url is None:
+ command.error("--direct-api-key-env requires --direct-base-url and --direct-model")
+ try:
+ binaries = resolve_binaries(
+ (
+ RequiredBinary(
+ "oha", "OHA_BIN", args.oha_bin, "Install oha with: cargo install oha"
+ ),
+ RequiredBinary(
+ "AIPerf",
+ "AIPERF_BIN",
+ args.aiperf_bin,
+ "Install AIPerf with: uv tool install --python 3.12 'aiperf==0.11.0'",
+ ),
+ RequiredBinary(
+ "switchyard-soak",
+ "SWITCHYARD_SOAK_BIN",
+ args.soak_bin,
+ "Build the scenario exporter with: cargo build --release -p switchyard-soak",
+ ),
+ )
+ )
+ output_dir = run_benchmark(
+ BenchmarkConfig(
+ base_url=args.base_url,
+ models=tuple(args.models),
+ concurrency=args.concurrency,
+ request_count=args.request_count,
+ tokenizer=args.tokenizer,
+ scenario_set=args.scenario_set,
+ scenarios=tuple(args.scenario),
+ load_profiles=tuple(args.load_profile or ["fixed"]),
+ profile_runs=args.profile_runs,
+ prompt_bytes=args.prompt_bytes,
+ max_output_tokens=args.max_output_tokens,
+ context_window_tokens=args.context_window_tokens,
+ request_rate=args.request_rate,
+ backend_label=args.backend_label,
+ output_dir=(args.output_dir or default_output_dir()).resolve(),
+ oha_bin=binaries["oha"],
+ aiperf_bin=binaries["AIPerf"],
+ soak_bin=binaries["switchyard-soak"],
+ scenario_backend_reset_url=args.scenario_backend_reset_url,
+ direct_baseline=(
+ DirectBaseline(
+ args.direct_base_url,
+ args.direct_model,
+ args.direct_api_key_env,
+ )
+ if args.direct_base_url is not None and args.direct_model is not None
+ else None
+ ),
+ )
+ )
+ except (OSError, RuntimeError) as error:
+ print(f"routing algorithm benchmark failed: {error}", file=sys.stderr)
+ return 1
+ print(f"Routing algorithm benchmark passed; report: {output_dir / 'report.md'}")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/local_soak_test.toml b/scripts/local_soak_test.toml
new file mode 100644
index 000000000..967095094
--- /dev/null
+++ b/scripts/local_soak_test.toml
@@ -0,0 +1,57 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Local routes used by run_local_soak_test.py; no provider credentials are required.
+
+schema_version = 1
+
+[llm_clients.mock]
+format = "openai_chat"
+base_url = "http://127.0.0.1:8100/v1"
+max_retries = 10
+
+[targets.classifier]
+id = "mock/classifier"
+llm_client = "mock"
+
+[targets.strong]
+id = "mock/strong"
+llm_client = "mock"
+
+[targets.weak]
+id = "mock/weak"
+llm_client = "mock"
+
+[routes.noop]
+id = "switchyard/noop"
+type = "noop"
+
+[routes.random]
+id = "switchyard/random"
+type = "random"
+targets = ["weak", "strong"]
+weights = [1, 0]
+seed = 42
+
+[routes.passthrough]
+id = "switchyard/passthrough"
+type = "passthrough"
+target = "weak"
+
+[routes.classifier]
+id = "switchyard/classifier"
+type = "llm_classifier"
+mode = "capability"
+classifier_target = "classifier"
+strong_target = "strong"
+weak_target = "weak"
+base_threshold = 1.0
+max_output_tokens = 1
+
+[routes.stage]
+id = "switchyard/stage"
+type = "stage_router"
+capable_target = "strong"
+efficient_target = "weak"
+picker = "efficient_first"
+confidence_threshold = 1.0
diff --git a/scripts/routing_overhead_plot.py b/scripts/routing_overhead_plot.py
new file mode 100644
index 000000000..65c6fd2a8
--- /dev/null
+++ b/scripts/routing_overhead_plot.py
@@ -0,0 +1,291 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Render routing-overhead results as a dependency-free SVG heatmap."""
+
+import math
+from collections.abc import Sequence
+from dataclasses import dataclass
+from html import escape
+from pathlib import Path
+from typing import TypeGuard
+
+
+@dataclass(frozen=True)
+class OverheadPlotRow:
+ """One route and workload comparison against the direct backend."""
+
+ scenario: str
+ load: str
+ route: str
+ ttft_delta_ms: float | None
+ ttft_delta_pct: float | None
+ token_throughput_delta: float | None
+ token_throughput_delta_pct: float | None
+
+
+@dataclass(frozen=True)
+class _Panel:
+ metric: str
+ title: str
+ subtitle: str
+ primary_suffix: str
+ primary_values: tuple[float | None, ...]
+ percent_values: tuple[float | None, ...]
+ higher_is_better: bool
+
+
+_BACKGROUND = (255, 255, 255)
+_BETTER = (42, 111, 174)
+_WORSE = (197, 52, 52)
+_NEUTRAL = "#f3f5f7"
+
+
+def _finite(value: float | None) -> TypeGuard[float]:
+ return value is not None and math.isfinite(value)
+
+
+def _fill(value: float | None, maximum: float, higher_is_better: bool) -> str:
+ if not _finite(value) or value == 0:
+ return _NEUTRAL
+ worse = value < 0 if higher_is_better else value > 0
+ target = _WORSE if worse else _BETTER
+ intensity = 0.16 + 0.64 * min(abs(value) / maximum, 1.0)
+ channels = (
+ round(background + (foreground - background) * intensity)
+ for background, foreground in zip(_BACKGROUND, target, strict=True)
+ )
+ red, green, blue = channels
+ return f"#{red:02x}{green:02x}{blue:02x}"
+
+
+def _format_value(value: float | None, suffix: str) -> str:
+ if not _finite(value):
+ return "n/a"
+ return f"{value:+,.2f}{suffix}"
+
+
+def _text(
+ x: float,
+ y: float,
+ value: str,
+ anchor: str = "start",
+ size: int = 13,
+ weight: int = 400,
+ fill: str = "#1d2733",
+) -> str:
+ return (
+ f'<text x="{x}" y="{y}" text-anchor="{anchor}" font-family="system-ui, '
+ f'-apple-system, sans-serif" font-size="{size}" font-weight="{weight}" '
+ f'fill="{fill}">{escape(value)}</text>'
+ )
+
+
+def write_overhead_plot(path: Path, rows: Sequence[OverheadPlotRow]) -> None:
+ """Write an annotated SVG overview for every supplied route and workload."""
+ if not rows:
+ raise ValueError("an overhead plot requires at least one row")
+
+ workloads = list(dict.fromkeys((row.scenario, row.load) for row in rows))
+ routes = list(dict.fromkeys(row.route for row in rows))
+ by_key: dict[tuple[str, str, str], OverheadPlotRow] = {}
+ for row in rows:
+ key = (row.scenario, row.load, row.route)
+ if key in by_key:
+ raise RuntimeError(
+ f"duplicate overhead plot row for {row.scenario}, {row.load}, and {row.route}"
+ )
+ by_key[key] = row
+
+ ordered_rows = [
+ by_key.get((scenario, load, route)) for scenario, load in workloads for route in routes
+ ]
+ panels = (
+ _Panel(
+ "ttft_p50_delta_ms",
+ "TTFT p50 overhead",
+ "Each cell shows the absolute change in ms and the percent change; lower is better",
+ " ms",
+ tuple(row.ttft_delta_ms if row is not None else None for row in ordered_rows),
+ tuple(row.ttft_delta_pct if row is not None else None for row in ordered_rows),
+ False,
+ ),
+ _Panel(
+ "output_tokens_per_second_delta",
+ "Output token throughput overhead",
+ "Each cell shows the absolute change in tokens/s and the percent change; higher is better",
+ " tokens/s",
+ tuple(row.token_throughput_delta if row is not None else None for row in ordered_rows),
+ tuple(
+ row.token_throughput_delta_pct if row is not None else None for row in ordered_rows
+ ),
+ True,
+ ),
+ )
+
+ label_width = 250
+ cell_width = 170
+ row_height = 50
+ panel_header_height = 76
+ panel_gap = 52
+ top = 112
+ right = 24
+ panel_height = panel_header_height + len(workloads) * row_height
+ width = label_width + len(routes) * cell_width + right
+ height = top + len(panels) * panel_height + panel_gap + 28
+ center = width / 2
+ legend_x = center - 260
+
+ svg = [
+ (
+ f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" '
+ f'viewBox="0 0 {width} {height}" role="img" '
+ 'aria-labelledby="routing-overhead-title routing-overhead-description">'
+ ),
+ '<title id="routing-overhead-title">Routing overhead versus the direct backend</title>',
+ (
+ '<desc id="routing-overhead-description">Two annotated heatmaps compare median '
+ "time to first token in milliseconds and output token throughput in tokens per "
+ "second for each route and workload. Every cell also gives the percent change. Red "
+ "cells are worse than the direct backend and blue cells are better. Darker cells "
+ "show a larger percent change within each heatmap.</desc>"
+ ),
+ f'<rect width="{width}" height="{height}" fill="#ffffff"/>',
+ _text(
+ center,
+ 31,
+ "Routing overhead versus the direct backend",
+ anchor="middle",
+ size=22,
+ weight=700,
+ ),
+ _text(
+ center,
+ 55,
+ "Measures Switchyard overhead when both arms use the same deployment, model, and settings",
+ anchor="middle",
+ size=13,
+ fill="#596777",
+ ),
+ f'<rect x="{legend_x}" y="72" width="18" height="14" rx="2" fill="#e6a9a9"/>',
+ _text(legend_x + 24, 84, "worse", size=12, fill="#596777"),
+ f'<rect x="{legend_x + 80}" y="72" width="18" height="14" rx="2" fill="#a8c7e2"/>',
+ _text(legend_x + 104, 84, "better", size=12, fill="#596777"),
+ f'<rect x="{legend_x + 163}" y="72" width="18" height="14" rx="2" fill="#f3f5f7"/>',
+ _text(
+ legend_x + 187,
+ 84,
+ "no change / unavailable; darker = larger % change",
+ size=12,
+ fill="#596777",
+ ),
+ ]
+
+ for panel_index, panel in enumerate(panels):
+ panel_y = top + panel_index * (panel_height + panel_gap)
+ svg.append(_text(center, panel_y + 18, panel.title, anchor="middle", size=16, weight=650))
+ svg.append(
+ _text(
+ center,
+ panel_y + 39,
+ panel.subtitle,
+ anchor="middle",
+ size=12,
+ fill="#596777",
+ )
+ )
+ svg.append(
+ _text(
+ label_width / 2,
+ panel_y + 66,
+ "Workload / load",
+ anchor="middle",
+ size=12,
+ weight=600,
+ )
+ )
+ for route_index, route in enumerate(routes):
+ cell_x = label_width + route_index * cell_width
+ svg.append(
+ _text(
+ cell_x + cell_width / 2,
+ panel_y + 66,
+ route.replace("-", " ").replace("_", " "),
+ anchor="middle",
+ size=12,
+ weight=600,
+ )
+ )
+
+ finite_values = [abs(value) for value in panel.percent_values if _finite(value)]
+ maximum = max(finite_values, default=1.0) or 1.0
+ row_y = panel_y + panel_header_height
+ for workload_index, (scenario, load) in enumerate(workloads):
+ y = row_y + workload_index * row_height
+ svg.append(
+ _text(
+ label_width / 2,
+ y + 21,
+ scenario.replace("-", " "),
+ anchor="middle",
+ size=12,
+ fill="#344150",
+ )
+ )
+ svg.append(
+ _text(
+ label_width / 2,
+ y + 38,
+ load.replace("-", " "),
+ anchor="middle",
+ size=11,
+ fill="#697786",
+ )
+ )
+ for route_index in range(len(routes)):
+ value_index = workload_index * len(routes) + route_index
+ primary_value = panel.primary_values[value_index]
+ percent_value = panel.percent_values[value_index]
+ x = label_width + route_index * cell_width
+ cell_title = (
+ f"{scenario}, {load}, {routes[route_index]}: "
+ f"{_format_value(primary_value, panel.primary_suffix)} "
+ f"({_format_value(percent_value, '%')})"
+ )
+ svg.append(
+ f'<g data-scenario="{escape(scenario, quote=True)}" '
+ f'data-load="{escape(load, quote=True)}" '
+ f'data-route="{escape(routes[route_index], quote=True)}" '
+ f'data-metric="{panel.metric}">'
+ )
+ svg.append(f"<title>{escape(cell_title)}</title>")
+ svg.append(
+ f'<rect x="{x + 2}" y="{y + 2}" width="{cell_width - 4}" '
+ f'height="{row_height - 4}" rx="3" '
+ f'fill="{_fill(percent_value, maximum, panel.higher_is_better)}" '
+ 'stroke="#d9e0e7"/>'
+ )
+ svg.append(
+ _text(
+ x + cell_width / 2,
+ y + 21,
+ _format_value(primary_value, panel.primary_suffix),
+ anchor="middle",
+ size=12,
+ weight=600,
+ )
+ )
+ svg.append(
+ _text(
+ x + cell_width / 2,
+ y + 38,
+ f"({_format_value(percent_value, '%')})",
+ anchor="middle",
+ size=11,
+ fill="#44515f",
+ )
+ )
+ svg.append("</g>")
+
+ svg.append("</svg>")
+ path.write_text("\n".join(svg) + "\n", encoding="utf-8")
diff --git a/scripts/run_local_soak_test.py b/scripts/run_local_soak_test.py
new file mode 100755
index 000000000..32decdae3
--- /dev/null
+++ b/scripts/run_local_soak_test.py
@@ -0,0 +1,369 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Run every Switchyard route with the local scenario backend and load tools."""
+
+import argparse
+import json
+import os
+import shutil
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.request
+from collections.abc import Sequence
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import TextIO
+
+from benchmark_routing_algorithms import (
+ BenchmarkConfig,
+ DirectBaseline,
+ RequiredBinary,
+ positive_int,
+ resolve_binaries,
+ run_benchmark,
+ run_checked,
+)
+
+ROUTES = (
+ ("noop", "switchyard/noop"),
+ ("random", "switchyard/random"),
+ ("passthrough", "switchyard/passthrough"),
+ ("llm_classifier", "switchyard/classifier"),
+ ("stage_router", "switchyard/stage"),
+)
+MOCK_PORT = 8100
+
+
+@dataclass
+class Child:
+ """A child process and the log file that must stay open while it runs."""
+
+ name: str
+ process: subprocess.Popen[str]
+ log: TextIO
+ log_path: Path
+
+
+def nonnegative_int(value: str) -> int:
+ """Parse a command-line integer that may be zero."""
+ parsed = int(value)
+ if parsed < 0:
+ raise argparse.ArgumentTypeError("must be zero or greater")
+ return parsed
+
+
+def tcp_port(value: str) -> int:
+ """Parse a valid nonzero TCP port."""
+ parsed = positive_int(value)
+ if parsed > 65_535:
+ raise argparse.ArgumentTypeError("must be between 1 and 65535")
+ return parsed
+
+
+def parser() -> argparse.ArgumentParser:
+ """Build the plain-English command-line interface."""
+ command = argparse.ArgumentParser(
+ description=(
+ "Start the request-aware scenario backend and a local Switchyard server, send one "
+ "request through every route, then run oha, AIPerf, and switchyard-soak."
+ ),
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ )
+ command.add_argument(
+ "--duration",
+ default="10s",
+ help="time to run switchyard-soak; use an s, m, or h suffix",
+ )
+ command.add_argument(
+ "--concurrency",
+ type=positive_int,
+ default=4,
+ help="concurrent requests used by oha, AIPerf, and the soak run",
+ )
+ command.add_argument(
+ "--request-count",
+ type=positive_int,
+ default=100,
+ help="measured requests each load tool sends for each routing algorithm",
+ )
+ command.add_argument(
+ "--mock-latency-ms",
+ type=nonnegative_int,
+ default=40,
+ help="latency before the local scenario backend sends its first token",
+ )
+ command.add_argument(
+ "--mock-token-latency-ms",
+ type=nonnegative_int,
+ default=1,
+ help="latency the local scenario backend adds between streamed output tokens",
+ )
+ command.add_argument(
+ "--server-port",
+ type=tcp_port,
+ default=4000,
+ help="local TCP port used by switchyard-server",
+ )
+ command.add_argument(
+ "--output-dir",
+ type=Path,
+ help="new directory for generated config, logs, and tool results",
+ )
+ return command
+
+
+def start_child(name: str, command: Sequence[str], log_path: Path) -> Child:
+ """Start one long-running process and retain its combined log."""
+ print(f"Starting {name}; log: {log_path}")
+ log = log_path.open("w", encoding="utf-8")
+ try:
+ process = subprocess.Popen(
+ list(command),
+ stdout=log,
+ stderr=subprocess.STDOUT,
+ text=True,
+ )
+ except OSError:
+ log.close()
+ raise
+ return Child(name=name, process=process, log=log, log_path=log_path)
+
+
+def stop_child(child: Child) -> None:
+ """Stop one child and close its log, escalating to kill after five seconds."""
+ if child.process.poll() is None:
+ child.process.terminate()
+ try:
+ child.process.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ child.process.kill()
+ child.process.wait(timeout=5)
+ child.log.close()
+
+
+def wait_for_health(child: Child, url: str, expected_status: str | None = None) -> None:
+ """Wait up to 30 seconds for a child process's health endpoint."""
+ for _ in range(120):
+ if child.process.poll() is not None:
+ raise RuntimeError(
+ f"{child.name} exited with status {child.process.returncode}; see {child.log_path}"
+ )
+ try:
+ with urllib.request.urlopen(url, timeout=1) as response:
+ body = response.read()
+ document = json.loads(body) if expected_status is not None else None
+ status = document.get("status") if isinstance(document, dict) else None
+ if response.status == 200 and (
+ expected_status is None or status == expected_status
+ ):
+ return
+ except (OSError, urllib.error.URLError, json.JSONDecodeError):
+ pass
+ time.sleep(0.25)
+ raise RuntimeError(f"{child.name} did not become healthy at {url}; see {child.log_path}")
+
+
+def check_routes(base_url: str, output_path: Path) -> None:
+ """Send one HTTP request through every configured route."""
+ records = []
+ for _algorithm, route in ROUTES:
+ body = json.dumps(
+ {
+ "model": route,
+ "messages": [{"role": "user", "content": "Reply with exactly OK."}],
+ "max_tokens": 8,
+ "stream": False,
+ }
+ ).encode()
+ request = urllib.request.Request(
+ f"{base_url}/v1/chat/completions",
+ data=body,
+ headers={"content-type": "application/json"},
+ method="POST",
+ )
+ with urllib.request.urlopen(request, timeout=30) as response:
+ records.append(
+ {
+ "route": route,
+ "status": response.status,
+ "selected_model": response.headers.get("x-model-router-selected-model"),
+ "response": json.loads(response.read()),
+ }
+ )
+ output_path.write_text(
+ "".join(f"{json.dumps(record)}\n" for record in records), encoding="utf-8"
+ )
+
+
+def default_output_dir(repo_root: Path) -> Path:
+ """Choose a new timestamped directory under the repository."""
+ stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+ return repo_root / "local-soak-test-results" / stamp
+
+
+def run_local_soak_test(args: argparse.Namespace) -> Path:
+ """Run the complete local soak test and return its output directory."""
+ repo_root = Path(__file__).resolve().parents[1]
+ rust_build = (
+ "Build the Rust commands with: cargo build --release -p switchyard-server "
+ "-p switchyard-soak --bins --example switchyard-soak-mock"
+ )
+ binaries = resolve_binaries(
+ (
+ RequiredBinary(
+ "switchyard-server",
+ "SWITCHYARD_SERVER_BIN",
+ os.environ.get(
+ "SWITCHYARD_SERVER_BIN",
+ str(repo_root / "target/release/switchyard-server"),
+ ),
+ rust_build,
+ ),
+ RequiredBinary(
+ "switchyard-soak",
+ "SWITCHYARD_SOAK_BIN",
+ os.environ.get(
+ "SWITCHYARD_SOAK_BIN", str(repo_root / "target/release/switchyard-soak")
+ ),
+ rust_build,
+ ),
+ RequiredBinary(
+ "scenario backend",
+ "SWITCHYARD_SOAK_MOCK_BIN",
+ os.environ.get(
+ "SWITCHYARD_SOAK_MOCK_BIN",
+ str(repo_root / "target/release/examples/switchyard-soak-mock"),
+ ),
+ rust_build,
+ ),
+ RequiredBinary(
+ "oha",
+ "OHA_BIN",
+ os.environ.get("OHA_BIN", "oha"),
+ "Install oha with: cargo install oha",
+ ),
+ RequiredBinary(
+ "AIPerf",
+ "AIPERF_BIN",
+ os.environ.get("AIPERF_BIN", "aiperf"),
+ "Install AIPerf with: uv tool install --python 3.12 'aiperf==0.11.0'",
+ ),
+ )
+ )
+ server_bin = binaries["switchyard-server"]
+ soak_bin = binaries["switchyard-soak"]
+ mock_bin = binaries["scenario backend"]
+ oha_bin = binaries["oha"]
+ aiperf_bin = binaries["AIPerf"]
+
+ output_dir = (args.output_dir or default_output_dir(repo_root)).resolve()
+ output_dir.mkdir(parents=True, exist_ok=False)
+ config_path = output_dir / "routes.toml"
+ shutil.copy2(repo_root / "scripts/local_soak_test.toml", config_path)
+
+ run_checked(
+ "switchyard-server config validation",
+ [server_bin, "--config", str(config_path), "--dry-run"],
+ output_dir / "config-validation.log",
+ )
+
+ children: list[Child] = []
+ try:
+ mock = start_child(
+ "scenario backend",
+ [
+ mock_bin,
+ "--port",
+ str(MOCK_PORT),
+ "--latency-ms",
+ str(args.mock_latency_ms),
+ "--token-latency-ms",
+ str(args.mock_token_latency_ms),
+ ],
+ output_dir / "scenario-backend.log",
+ )
+ children.append(mock)
+ wait_for_health(mock, f"http://127.0.0.1:{MOCK_PORT}/health")
+
+ server = start_child(
+ "switchyard-server",
+ [server_bin, "--config", str(config_path), "--port", str(args.server_port)],
+ output_dir / "switchyard-server.log",
+ )
+ children.append(server)
+ base_url = f"http://127.0.0.1:{args.server_port}"
+ wait_for_health(server, f"{base_url}/health", "ok")
+ check_routes(base_url, output_dir / "route-checks.jsonl")
+
+ run_benchmark(
+ BenchmarkConfig(
+ base_url=base_url,
+ models=ROUTES,
+ concurrency=args.concurrency,
+ request_count=args.request_count,
+ backend_label=(
+ "request-aware local backend with "
+ f"{args.mock_latency_ms} ms TTFT and "
+ f"{args.mock_token_latency_ms} ms per output token"
+ ),
+ output_dir=output_dir / "routing-benchmark",
+ oha_bin=oha_bin,
+ aiperf_bin=aiperf_bin,
+ soak_bin=soak_bin,
+ profile_runs=1,
+ scenario_backend_reset_url=f"http://127.0.0.1:{MOCK_PORT}/reset",
+ direct_baseline=DirectBaseline(
+ base_url=f"http://127.0.0.1:{MOCK_PORT}",
+ model="mock/weak",
+ ),
+ )
+ )
+
+ run_checked(
+ "switchyard-soak API and streaming requests",
+ [
+ soak_bin,
+ "--base-url",
+ base_url,
+ "--model",
+ "switchyard/passthrough",
+ "--duration",
+ args.duration,
+ "--concurrency",
+ str(args.concurrency),
+ "--report-interval",
+ "2",
+ "--invalid-canary-interval",
+ "0",
+ "--server-pid",
+ str(server.process.pid),
+ "--results-dir",
+ str(output_dir / "soak"),
+ ],
+ output_dir / "soak.log",
+ )
+ finally:
+ for child in reversed(children):
+ stop_child(child)
+
+ return output_dir
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ """Parse arguments, run the local soak test, and print one actionable failure."""
+ args = parser().parse_args(argv)
+ try:
+ output_dir = run_local_soak_test(args)
+ except (OSError, RuntimeError) as error:
+ print(f"local soak test failed: {error}", file=sys.stderr)
+ return 1
+ print(f"Local soak test passed; results: {output_dir}")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/test_aiperf_runner.py b/tests/test_aiperf_runner.py
new file mode 100644
index 000000000..71f5fafa7
--- /dev/null
+++ b/tests/test_aiperf_runner.py
@@ -0,0 +1,157 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import sys
+import time
+
+import pytest
+
+import scripts.aiperf_runner as aiperf_runner
+from scripts.aiperf_runner import aggregate_exports, run_profile, validate_aiperf_version
+
+
+def _write_stubborn_worker(worker) -> None:
+ worker.write_text(
+ """import signal
+import sys
+import time
+from pathlib import Path
+
+signal.signal(signal.SIGTERM, signal.SIG_IGN)
+heartbeat = Path(sys.argv[1])
+while True:
+ heartbeat.write_text(str(time.monotonic()))
+ time.sleep(0.05)
+"""
+ )
+
+
+def _assert_heartbeat_stopped(heartbeat) -> None:
+ heartbeat_before = heartbeat.read_text()
+ time.sleep(0.2)
+ assert heartbeat.read_text() == heartbeat_before
+
+
+def test_run_profile_timeout_kills_the_worker_process_group(tmp_path, monkeypatch) -> None:
+ fake = tmp_path / "fake_aiperf.py"
+ worker = tmp_path / "worker.py"
+ child_pid = tmp_path / "child-pid"
+ heartbeat = tmp_path / "heartbeat"
+ _write_stubborn_worker(worker)
+ monkeypatch.setattr(aiperf_runner, "PROCESS_GROUP_GRACE_SECONDS", 0.2)
+ fake.write_text(
+ """import subprocess
+import sys
+import time
+from pathlib import Path
+
+child = subprocess.Popen([sys.executable, sys.argv[2], sys.argv[3]])
+Path(sys.argv[1]).write_text(str(child.pid))
+time.sleep(60)
+"""
+ )
+
+ started_at = time.monotonic()
+ with pytest.raises(RuntimeError, match="exceeded its 1-second run limit"):
+ run_profile(
+ [sys.executable, str(fake), str(child_pid), str(worker), str(heartbeat)],
+ tmp_path / "aiperf.log",
+ tmp_path / "artifacts",
+ timeout_seconds=1,
+ )
+
+ assert time.monotonic() - started_at < 10
+ assert child_pid.read_text().isdigit()
+ _assert_heartbeat_stopped(heartbeat)
+ assert "exceeded its 1-second run limit" in (tmp_path / "aiperf.log").read_text()
+
+
+def test_run_profile_stops_workers_after_the_leader_exits(tmp_path, monkeypatch) -> None:
+ fake = tmp_path / "fake_aiperf.py"
+ worker = tmp_path / "worker.py"
+ child_pid = tmp_path / "child-pid"
+ heartbeat = tmp_path / "heartbeat"
+ _write_stubborn_worker(worker)
+ monkeypatch.setattr(aiperf_runner, "PROCESS_GROUP_GRACE_SECONDS", 0.2)
+ fake.write_text(
+ """import subprocess
+import sys
+import time
+from pathlib import Path
+
+child = subprocess.Popen([sys.executable, sys.argv[2], sys.argv[3]])
+Path(sys.argv[1]).write_text(str(child.pid))
+while not Path(sys.argv[3]).exists():
+ time.sleep(0.01)
+"""
+ )
+
+ with pytest.raises(RuntimeError, match="did not write profile_export_aiperf.json"):
+ run_profile(
+ [sys.executable, str(fake), str(child_pid), str(worker), str(heartbeat)],
+ tmp_path / "aiperf.log",
+ tmp_path / "artifacts",
+ timeout_seconds=10,
+ )
+
+ assert child_pid.read_text().isdigit()
+ _assert_heartbeat_stopped(heartbeat)
+
+
+def test_process_group_probe_ignores_an_unowned_reused_group(monkeypatch) -> None:
+ def deny_signal(_process_group, _signal) -> None:
+ raise PermissionError
+
+ monkeypatch.setattr(aiperf_runner.os, "killpg", deny_signal)
+
+ assert not aiperf_runner._process_group_exists(1234)
+
+
+def test_validate_aiperf_version_rejects_uncovered_release(tmp_path) -> None:
+ fake = tmp_path / "aiperf"
+ fake.write_text("#!/bin/sh\nprintf '0.12.0\\n'\n")
+ fake.chmod(0o755)
+
+ with pytest.raises(RuntimeError, match="AIPerf 0.12.0 is unsupported"):
+ validate_aiperf_version(str(fake))
+
+
+def test_aggregate_exports_combines_independent_runs(tmp_path) -> None:
+ exports = []
+ for index, throughput in enumerate((10.0, 14.0), start=1):
+ path = tmp_path / f"run-{index}.json"
+ document = {
+ "aiperf_version": "0.11.0",
+ "error_request_count": None,
+ "request_count": {"unit": "requests", "avg": 20},
+ "request_throughput": {
+ "unit": "requests/sec",
+ "avg": throughput,
+ },
+ "time_to_first_token": {"unit": "ms", "p50": 10.0},
+ }
+ if index == 2:
+ del document["error_request_count"]
+ document["error_summary"] = []
+ path.write_text(json.dumps(document))
+ exports.append(path)
+
+ aggregate_path = aggregate_exports(exports, tmp_path / "aggregate" / "report.json")
+ metrics = json.loads(aggregate_path.read_text())["metrics"]
+
+ assert metrics["request_throughput_avg"]["mean"] == 12.0
+ assert metrics["request_throughput_avg"]["ci_low"] < 0
+ assert metrics["error_request_count_avg"]["mean"] == 0.0
+
+ mismatched = json.loads(exports[1].read_text())
+ mismatched["time_to_first_token"]["unit"] = "seconds"
+ exports[1].write_text(json.dumps(mismatched))
+ with pytest.raises(RuntimeError, match="disagree on the unit"):
+ aggregate_exports(exports, tmp_path / "invalid" / "report.json")
+
+ mismatched["time_to_first_token"]["unit"] = "ms"
+ mismatched["request_count"]["avg"] = True
+ exports[1].write_text(json.dumps(mismatched))
+ with pytest.raises(RuntimeError, match="has no numeric request_count.avg"):
+ aggregate_exports(exports, tmp_path / "malformed" / "report.json")
diff --git a/tests/test_routing_overhead_plot.py b/tests/test_routing_overhead_plot.py
new file mode 100644
index 000000000..9a7ddf42e
--- /dev/null
+++ b/tests/test_routing_overhead_plot.py
@@ -0,0 +1,64 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import xml.etree.ElementTree as ET
+
+from scripts.routing_overhead_plot import OverheadPlotRow, write_overhead_plot
+
+
+def test_overhead_plot_preserves_matrix_and_unavailable_values(tmp_path) -> None:
+ path = tmp_path / "routing-overhead.svg"
+ rows = (
+ OverheadPlotRow("short-<interactive>", "fixed", "random", 3.0, 25.0, -80.0, -11.43),
+ OverheadPlotRow("short-<interactive>", "fixed", "stage&router", None, None, -14.0, -2.0),
+ OverheadPlotRow("long-context", "traffic-burst", "random", 1.25, 3.0, None, None),
+ OverheadPlotRow("long-context", "traffic-burst", "stage&router", -0.5, -1.0, 7.0, 1.0),
+ )
+
+ write_overhead_plot(path, rows)
+
+ root = ET.fromstring(path.read_text())
+ namespace = {"svg": "http://www.w3.org/2000/svg"}
+ cells = root.findall(".//svg:g", namespace)
+ text_nodes = root.findall(".//svg:text", namespace)
+ values = {
+ (
+ cell.attrib["data-scenario"],
+ cell.attrib["data-load"],
+ cell.attrib["data-route"],
+ cell.attrib["data-metric"],
+ ): " ".join("".join(cell.itertext()).split())
+ for cell in cells
+ }
+ assert len(values) == 8
+ assert (
+ values[
+ (
+ "short-<interactive>",
+ "fixed",
+ "random",
+ "output_tokens_per_second_delta",
+ )
+ ]
+ == "short-<interactive>, fixed, random: -80.00 tokens/s (-11.43%) "
+ "-80.00 tokens/s (-11.43%)"
+ )
+ assert (
+ "n/a"
+ in values[("long-context", "traffic-burst", "random", "output_tokens_per_second_delta")]
+ )
+ assert "n/a" in values[("short-<interactive>", "fixed", "stage&router", "ttft_p50_delta_ms")]
+ centered_labels = {
+ node.text: node.attrib["text-anchor"]
+ for node in text_nodes
+ if node.text in {"Routing overhead versus the direct backend", "short <interactive>"}
+ }
+ assert centered_labels == {
+ "Routing overhead versus the direct backend": "middle",
+ "short <interactive>": "middle",
+ }
+ assert all(
+ text.attrib["text-anchor"] == "middle"
+ for cell in cells
+ for text in cell.findall("svg:text", namespace)
+ )
diff --git a/tests/test_routing_performance_report.py b/tests/test_routing_performance_report.py
new file mode 100644
index 000000000..445357cc1
--- /dev/null
+++ b/tests/test_routing_performance_report.py
@@ -0,0 +1,391 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import csv
+import json
+from dataclasses import replace
+from pathlib import Path
+
+import pytest
+
+import scripts.benchmark_routing_algorithms as benchmark
+from scripts.benchmark_routing_algorithms import (
+ BenchmarkArm,
+ BenchmarkConfig,
+ DirectBaseline,
+ ScenarioDefinition,
+ compare_to_direct_backend,
+ materialize_aiperf_input,
+ parse_result,
+ profile_request_count,
+ write_report,
+)
+
+
+def test_report_combines_scenarios_single_and_aggregate_results(tmp_path) -> None:
+ oha_path = tmp_path / "oha.json"
+ single_path = tmp_path / "profile_export_aiperf.json"
+ direct_path = tmp_path / "profile_export_aiperf_direct.json"
+ aggregate_path = tmp_path / "profile_export_aiperf_aggregate.json"
+ oha_path.write_text(
+ json.dumps(
+ {
+ "summary": {"successRate": 1.0, "requestsPerSec": 120.5},
+ "latencyPercentiles": {"p50": 0.012, "p99": 0.045},
+ }
+ )
+ )
+ single_path.write_text(
+ json.dumps(
+ {
+ "error_request_count": {"unit": "requests", "avg": 0},
+ "request_count": {"unit": "requests", "avg": 4},
+ "request_throughput": {"unit": "requests/sec", "avg": 95.5},
+ "request_latency": {"unit": "ms", "p50": 40.0},
+ "time_to_first_token": {"unit": "ms", "p50": 15.0, "p99": 35.0},
+ "output_token_throughput": {"unit": "tokens/sec", "avg": 620.0},
+ "e2e_output_token_throughput": {
+ "unit": "tokens/sec/user",
+ "p50": 200.0,
+ },
+ }
+ )
+ )
+ direct_path.write_text(
+ json.dumps(
+ {
+ "error_request_count": {"unit": "requests", "avg": 0},
+ "request_count": {"unit": "requests", "avg": 4},
+ "request_throughput": {"unit": "requests/sec", "avg": 100.0},
+ "request_latency": {"unit": "ms", "p50": 35.0},
+ "time_to_first_token": {"unit": "ms", "p50": 12.0, "p99": 30.0},
+ "output_token_throughput": {"unit": "tokens/sec", "avg": 700.0},
+ }
+ )
+ )
+ aggregate_path.write_text(
+ json.dumps(
+ {
+ "metrics": {
+ "error_request_count_avg": {"unit": "requests", "mean": 2},
+ "request_count_avg": {"unit": "requests", "mean": 4},
+ "request_throughput_avg": {
+ "unit": "requests/sec",
+ "mean": 80.0,
+ "cv": 0.03,
+ "ci_low": 77.0,
+ "ci_high": 83.0,
+ },
+ "request_latency_p50": {"unit": "seconds", "mean": 0.05},
+ "time_to_first_token_p50": {"unit": "ms", "mean": 20.0},
+ "time_to_first_token_p99": {"unit": "ms", "mean": 45.0},
+ "output_token_throughput_avg": {
+ "unit": "tokens/sec",
+ "mean": 500.0,
+ },
+ }
+ }
+ )
+ )
+ short = ScenarioDefinition(
+ id="short-interactive",
+ group="core",
+ description="short baseline",
+ expected="all requests succeed",
+ expected_error_rate_min=0.0,
+ expected_error_rate_max=0.0,
+ input_file=tmp_path / "short.json",
+ load_profiles=(),
+ )
+ failure = ScenarioDefinition(
+ id="failure-pressure",
+ group="resilience",
+ description="failure injection",
+ expected="failures stay bounded",
+ expected_error_rate_min=0.01,
+ expected_error_rate_max=0.75,
+ input_file=tmp_path / "failure.json",
+ load_profiles=(),
+ )
+ before = {
+ "models": {
+ "mock/weak": {"calls": 10, "errors": 0},
+ "mock/classifier": {"calls": 3, "errors": 0},
+ },
+ "classifier": {
+ "total_requests": 3,
+ "total_errors": 0,
+ "models": {
+ "mock/classifier": {
+ "calls": 3,
+ "errors": 0,
+ "model_call_latency": {"count": 3, "total_ms": 6},
+ }
+ },
+ },
+ "routing_overhead": {"count": 10, "total_ms": 20},
+ }
+ after = {
+ "models": {
+ "mock/weak": {"calls": 20, "errors": 2},
+ "mock/strong": {"calls": 2, "errors": 0},
+ "mock/classifier": {"calls": 5, "errors": 1},
+ },
+ "classifier": {
+ "total_requests": 5,
+ "total_errors": 1,
+ "models": {
+ "mock/classifier": {
+ "calls": 5,
+ "errors": 1,
+ "model_call_latency": {"count": 5, "total_ms": 14},
+ }
+ },
+ },
+ "routing_overhead": {"count": 22, "total_ms": 50},
+ }
+ results = compare_to_direct_backend(
+ (
+ parse_result(
+ BenchmarkArm(
+ "direct-backend",
+ "mock/weak",
+ "http://127.0.0.1:8100",
+ True,
+ ),
+ short,
+ "fixed",
+ None,
+ direct_path,
+ {},
+ {},
+ ),
+ parse_result(
+ BenchmarkArm(
+ "random",
+ "switchyard/random",
+ "http://127.0.0.1:4000",
+ False,
+ ),
+ short,
+ "fixed",
+ oha_path,
+ single_path,
+ before,
+ after,
+ ),
+ parse_result(
+ BenchmarkArm(
+ "classifier",
+ "switchyard/classifier",
+ "http://127.0.0.1:4000",
+ False,
+ ),
+ failure,
+ "fixed",
+ None,
+ aggregate_path,
+ before,
+ after,
+ ),
+ )
+ )
+ output_dir = tmp_path / "report"
+ output_dir.mkdir()
+ config = BenchmarkConfig(
+ base_url="http://127.0.0.1:4000",
+ models=(("random", "switchyard/random"),),
+ concurrency=100,
+ request_count=1000,
+ backend_label="test backend",
+ output_dir=output_dir,
+ oha_bin="oha",
+ aiperf_bin="aiperf",
+ soak_bin="switchyard-soak",
+ direct_baseline=DirectBaseline("http://127.0.0.1:8100", "mock/weak"),
+ )
+
+ write_report(config, results)
+
+ report = (output_dir / "report.md").read_text()
+ assert "## Routing overhead versus the direct backend" in report
+ assert "![Routing overhead plots](routing-overhead.svg)" in report
+ assert (
+ report.index("## Routing overhead versus the direct backend")
+ < report.index("## Run details")
+ < report.index("## Throughput and latency")
+ )
+ assert (
+ "| short interactive | random | fixed | +5.00 ms | +3.00 ms | +25.00% | "
+ "-4.50% | -80.00 tokens/s | -11.43% |" in report
+ )
+ assert "| short interactive | short baseline |" in report
+ assert "| random | short-interactive | fixed | 120.50 | 95.50 |" in report
+ assert "## Repeatability" in report
+ assert "## Resilience" in report
+ assert (
+ "| classifier | failure-pressure | fixed | 50.00% | 1.00%–75.00% | PASS | `{"
+ '"mock/weak":2}` | failures stay bounded |' in report
+ )
+ rows = list(csv.DictReader((output_dir / "report.csv").open()))
+ rows_by_key = {(row["algorithm"], row["scenario"]): row for row in rows}
+ random_row = rows_by_key[("random", "short-interactive")]
+ assert random_row["selected_model_calls"] == '{"mock/strong":2,"mock/weak":10}'
+ assert random_row["selected_model_share"] == '{"mock/strong":0.1667,"mock/weak":0.8333}'
+ assert random_row["selected_model_errors"] == '{"mock/weak":2}'
+ payload = json.loads((output_dir / "report.json").read_text())
+ payload_by_key = {(row["algorithm"], row["scenario"]): row for row in payload["results"]}
+ direct_row = payload_by_key[("direct-backend", "short-interactive")]
+ random_payload = payload_by_key[("random", "short-interactive")]
+ classifier_payload = payload_by_key[("classifier", "failure-pressure")]
+ assert direct_row["aiperf_itl_p50_ms"] is None
+ assert direct_row["aiperf_request_throughput_delta_pct"] is None
+ assert random_payload["aiperf_output_tokens_per_second_per_user_p50"] == 200.0
+ assert random_payload["aiperf_ttft_p50_delta_pct"] == 25.0
+ assert random_payload["aiperf_output_tokens_per_second_delta"] == -80.0
+ assert random_payload["scenario_description"] == "short baseline"
+ assert classifier_payload["aiperf_request_latency_p50_ms"] == 50.0
+ assert classifier_payload["aiperf_request_throughput_cv"] == 0.03
+ assert classifier_payload["classifier_latency_avg_ms"] == 4.0
+
+ assert (output_dir / "routing-overhead.svg").is_file()
+
+ with pytest.raises(RuntimeError, match="missing direct baseline for mixed-traffic fixed"):
+ compare_to_direct_backend((results[0], replace(results[1], scenario="mixed-traffic")))
+
+
+def test_materialized_sessions_are_unique_and_cover_request_count(tmp_path) -> None:
+ scenario_path = tmp_path / "short.json"
+ scenario_path.write_text(
+ json.dumps(
+ {
+ "data": [
+ {
+ "session_id": "short-0",
+ "payloads": [{"model": "test"}, {"model": "test"}],
+ }
+ ]
+ }
+ )
+ )
+ scenario = ScenarioDefinition(
+ id="short-interactive",
+ group="core",
+ description="short baseline",
+ expected="all requests succeed",
+ expected_error_rate_min=0.0,
+ expected_error_rate_max=0.0,
+ input_file=scenario_path,
+ load_profiles=(),
+ )
+
+ expanded_input = materialize_aiperf_input(scenario, tmp_path / "expanded.json", 5)
+ expanded_sessions = json.loads(expanded_input.read_text())["data"]
+
+ assert len(expanded_sessions) == 3
+ assert len({session["session_id"] for session in expanded_sessions}) == 3
+
+
+def test_traffic_burst_request_count_integrates_rate_series(tmp_path) -> None:
+ config = BenchmarkConfig(
+ base_url="http://127.0.0.1:4000",
+ models=(("random", "switchyard/random"),),
+ concurrency=2,
+ request_count=100,
+ backend_label="test backend",
+ output_dir=tmp_path,
+ oha_bin="oha",
+ aiperf_bin="aiperf",
+ soak_bin="switchyard-soak",
+ )
+ profile = {
+ "kind": "traffic_burst",
+ "duration_seconds": 30,
+ "points": [
+ {"time_s": 0, "rate_multiplier": 1},
+ {"time_s": 10, "rate_multiplier": 1},
+ {"time_s": 11, "rate_multiplier": 10},
+ {"time_s": 16, "rate_multiplier": 10},
+ {"time_s": 17, "rate_multiplier": 1},
+ {"time_s": 30, "rate_multiplier": 1},
+ ],
+ }
+
+ assert profile_request_count(config, profile) == 168
+
+
+def test_aiperf_cells_use_disjoint_artifacts(tmp_path, monkeypatch) -> None:
+ scenario_path = tmp_path / "short.json"
+ scenario_path.write_text(
+ json.dumps(
+ {
+ "data": [
+ {
+ "session_id": "short-0",
+ "payloads": [{"model": "switchyard/random"}],
+ }
+ ]
+ }
+ )
+ )
+ scenario = ScenarioDefinition(
+ id="short-interactive",
+ group="core",
+ description="short baseline",
+ expected="all requests succeed",
+ expected_error_rate_min=0.0,
+ expected_error_rate_max=0.0,
+ input_file=scenario_path,
+ load_profiles=(),
+ )
+ config = BenchmarkConfig(
+ base_url="http://127.0.0.1:4000",
+ models=(("random", "switchyard/random"),),
+ concurrency=1,
+ request_count=1,
+ backend_label="test backend",
+ output_dir=tmp_path,
+ oha_bin="oha",
+ aiperf_bin="aiperf",
+ soak_bin="switchyard-soak",
+ profile_runs=1,
+ )
+ arm = BenchmarkArm(
+ "random",
+ "switchyard/random",
+ "http://127.0.0.1:4000",
+ False,
+ )
+ observed: list[tuple[Path, Path]] = []
+
+ def fake_run_profile(
+ _command, log_path: Path, artifact_dir: Path, _timeout_seconds: int
+ ) -> Path:
+ observed.append((log_path, artifact_dir))
+ artifact_dir.mkdir(parents=True)
+ export = artifact_dir / "profile_export_aiperf.json"
+ export.write_text("{}")
+ return export
+
+ monkeypatch.setattr(benchmark, "run_profile", fake_run_profile)
+
+ first = benchmark.run_aiperf(
+ config,
+ arm,
+ scenario,
+ {"kind": "fixed"},
+ tmp_path,
+ "algorithm-00-short-interactive-fixed",
+ )
+ second = benchmark.run_aiperf(
+ config,
+ arm,
+ scenario,
+ {"kind": "fixed"},
+ tmp_path,
+ "algorithm-00-long-context-fixed",
+ )
+
+ assert first != second
+ assert observed[0][0] != observed[1][0]
+ assert observed[0][1] != observed[1][1]