Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(operations): add release soak and routing benchmarks (#176)
Elyas Mehtabuddin71eb274db6d6
diff --git a/.gitignore b/.gitignoreindex fdc44128c..c456be335 100644--- a/.gitignore+++ b/.gitignore@@ -150,6 +150,9 @@ plans/# Benchmark result directoriesbench_results*/+soak-results/+local-soak-test-results/+routing-benchmark-results/jobs/**/*.*pytest-of-*/diff --git a/Cargo.lock b/Cargo.lockindex 405dda9da..ff7a91185 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2388,6 +2388,23 @@ dependencies = ["tokio",]+[[package]]+name = "switchyard-soak"+version = "0.2.0"+dependencies = [+ "axum",+ "clap",+ "futures-util",+ "humantime",+ "parking_lot",+ "rand 0.10.2",+ "reqwest",+ "serde",+ "serde_json",+ "tempfile",+ "tokio",+]+[[package]]name = "switchyard-translation"version = "0.2.0"diff --git a/Cargo.toml b/Cargo.tomlindex 98433d8c1..d929d0c58 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -10,6 +10,7 @@ members = ["crates/protocol","crates/switchyard-server","crates/switchyard-skill-distillation",+ "crates/switchyard-soak","crates/switchyard-translation",]diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rsindex 95f4c7212..8a9e42c47 100644--- a/crates/libsy/src/algorithms/noop.rs+++ b/crates/libsy/src/algorithms/noop.rs@@ -23,11 +23,12 @@ impl Algorithm for Noop {}async fn route(self: Arc<Self>, _driver: Driver, request: Request) -> Result<RoutingOutcome> {+ let streaming = request.llm_request.stream;let model_id = request.model_id().unwrap_or_else(|| ModelId::from("switchyard/noop"));tracing::info!(target = %model_id, "noop returned its synthetic response");- let llm_response = LlmResponse::Agg(AggLlmResponse {+ let aggregate = AggLlmResponse {id: Some("switchyard-noop".to_string()),model: Some(model_id.to_string()),outputs: vec![ResponseOutput {@@ -38,7 +39,12 @@ impl Algorithm for Noop {stop_reason: Some(StopReason::EndTurn),}],..Default::default()- });+ };+ let llm_response = if streaming {+ LlmResponse::Stream(aggregate.into_stream())+ } else {+ LlmResponse::Agg(aggregate)+ };let response = Response {llm_response,metadata: request.metadata.clone(),@@ -75,4 +81,24 @@ mod tests {assert_eq!(response.selected_model(), Some(TEST_MODEL));Ok(())}++ #[tokio::test]+ async fn test_noop_algo_streams_when_requested() -> Result<()> {+ let request = Request {+ llm_request: LlmRequest {+ model: Some("streaming-noop".to_string()),+ messages: vec![Message::text(Role::User, "hi")],+ stream: true,+ ..LlmRequest::default()+ },+ raw_request: None,+ metadata: None,+ };++ let algorithm: Arc<dyn Algorithm> = Arc::new(Noop {});+ let (_, response) = test_drive(algorithm, request, echo()).await?;++ assert!(matches!(response.llm_response, LlmResponse::Stream(_)));+ Ok(())+ }}diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rsindex 420ac5e2b..0238bd041 100644--- a/crates/switchyard-server/src/config.rs+++ b/crates/switchyard-server/src/config.rs@@ -1676,6 +1676,21 @@ target = "azure"return Err(ServerError::new("primary llm client is missing"));};assert_eq!(primary.max_retries, 0);++ let maximum = VALID_CONFIG.replacen(+ "base_url = \"https://example.test/v1\"",+ &format!(+ "base_url = \"https://example.test/v1\"\nmax_retries = {MAX_CONFIGURED_RETRIES}"+ ),+ 1,+ );+ let config: ServerConfig = toml::from_str(&maximum).map_err(|error| {+ ServerError::new(format!("failed to parse maximum retry config: {error}"))+ })?;+ let Some(primary) = config.llm_clients.get("primary") else {+ return Err(ServerError::new("primary llm client is missing"));+ };+ assert_eq!(primary.max_retries, MAX_CONFIGURED_RETRIES);Ok(())}diff --git a/crates/switchyard-soak/Cargo.toml b/crates/switchyard-soak/Cargo.tomlnew file mode 100644index 000000000..242dbed17--- /dev/null+++ b/crates/switchyard-soak/Cargo.toml@@ -0,0 +1,37 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++[package]+name = "switchyard-soak"+version.workspace = true+description = "Sustained load tester for a live Switchyard server"+authors.workspace = true+edition.workspace = true+license.workspace = true+repository.workspace = true+rust-version.workspace = true+readme = "README.md"++[[bin]]+name = "switchyard-soak"+path = "src/main.rs"++[[example]]+name = "switchyard-soak-mock"+path = "examples/mock_server.rs"++[dependencies]+clap = { version = "4", features = ["derive"] }+futures-util.workspace = true+humantime = "2.4"+parking_lot.workspace = true+rand.workspace = true+reqwest.workspace = true+serde.workspace = true+serde_json.workspace = true+tokio.workspace = true++[dev-dependencies]+axum = "0.8"+tempfile = "3"+tokio.workspace = truediff --git a/crates/switchyard-soak/README.md b/crates/switchyard-soak/README.mdnew file mode 100644index 000000000..7f739f52f--- /dev/null+++ b/crates/switchyard-soak/README.md@@ -0,0 +1,119 @@+# switchyard-soak++`switchyard-soak` keeps a fixed number of requests in flight against one route on a live+`switchyard-server`. Its scenario catalog covers short and long inputs, long outputs, shared+prefixes, mixed traffic, growing conversations, tool schemas, tool-call bursts, routing signals,+and bounded failure cases. The short baseline also exercises Chat Completions, Messages, and+Responses in streaming and non-streaming form. The command samples health, metrics, and an+optional local server process, then exits with status 1 when a release gate fails.++Build the server and soak tester from the commit being tested:++```bash+cargo build --release -p switchyard-server -p switchyard-soak \+ --bins --example switchyard-soak-mock+```++Run a five-minute check against a route advertised by `GET /v1/models`:++```bash+target/release/switchyard-soak \+ --base-url http://127.0.0.1:4000 \+ --model switchyard/general \+ --duration 5m \+ --concurrency 4 \+ --report-interval 10+```++Run the 48-hour release gate and measure the local server process:++```bash+target/release/switchyard-soak \+ --model switchyard/general \+ --duration 48h \+ --concurrency 16 \+ --server-pid "$SWITCHYARD_SERVER_PID" \+ --max-rss-growth-mib 512+```++If the endpoint needs a bearer token, put the secret in an environment variable and pass the+variable's name:++```bash+export SWITCHYARD_SOAK_TOKEN="..."+target/release/switchyard-soak \+ --model switchyard/general \+ --api-key-env SWITCHYARD_SOAK_TOKEN+```++## Flags++| Flag | Default | What it does |+|---|---:|---|+| `--base-url URL` | `http://127.0.0.1:4000` | Sends every check and inference request to this Switchyard server. |+| `--model ID` | required | Selects one exact route id returned by `GET /v1/models`. The test stops before sending load if the id is missing. |+| `--duration TIME` | `48h` | Keeps the load running for this long. Use seconds, minutes, or hours, such as `30s`, `5m`, or `48h`. |+| `--concurrency N` | `16` | Keeps this many requests in flight. Each worker waits for its response before sending another request. |+| `--max-output-tokens N` | `32` | Sets the default output limit. `decode-heavy` uses its bounded 512-token and 1,024-token limits. |+| `--prompt-bytes N` | `1024` | Sizes the repeated input used by short and prefix-reuse scenarios. Larger values put more pressure on request memory and prefix caching. |+| `--scenario-set NAME` | `standard` | Selects `core`, `agentic`, `resilience`, `standard`, or `all`. `standard` includes core and agentic scenarios without expected failures. |+| `--scenario ID` | none | Selects one exact scenario id. Repeat the flag to select more scenarios; explicit ids take precedence over `--scenario-set`. |+| `--context-window-tokens N` | `32768` | Bounds generated long-context and overflow inputs. The command rejects values above 1,000,000. |+| `--export-scenarios PATH` | none | Writes a manifest and AIPerf `inputs-json` files, then exits without contacting a server. The performance script uses this mode. |+| `--request-timeout SECONDS` | `120` | Limits connection time and idle time between response bytes. A healthy stream may run longer if it keeps producing data. |+| `--report-interval SECONDS` | `60` | Sets how often the test samples health, metrics, RSS, CPU, and interval latency. |+| `--invalid-canary-interval SECONDS` | `300` | Sends malformed input this often, expects HTTP 400, and then checks recovery. Set it to `0` only when a permissive mock cannot reject the canary. |+| `--max-error-rate FRACTION` | `0` | Allows at most this fraction of inference requests to fail. `0.01` means one percent. The value must be between 0 and 1. |+| `--server-pid PID` | none | Samples RSS and CPU from this local `switchyard-server` process. Omit it for a remote server. |+| `--max-rss-growth-mib MIB` | none | Fails when the last RSS sample exceeds the first by more than this amount. It requires `--server-pid`. |+| `--api-key-env NAME` | none | Reads a bearer token from this environment variable without putting the secret in the command or result files. |+| `--results-dir PATH` | `soak-results/<UTC time>` | Creates this new directory for the run. The command refuses to reuse an existing directory. |+| `--help` | n/a | Prints the command reference and examples. |+| `--version` | n/a | Prints the crate version. |++The scenario source lives in `src/scenarios/`. Each request shape has one file and a pure builder.+The exported manifest is the only scenario input read by the Python performance runner, so model+ids, sessions, tools, and message histories cannot drift between the Rust soak and AIPerf runs.+Concurrency knees and traffic bursts are load profiles attached to the short baseline, not+duplicate request scenarios.++The catalog contains:++| Group | Scenarios |+|---|---|+| Core | `short-interactive`, `long-context`, `decode-heavy`, `prefix-reuse`, `mixed-traffic` |+| Agentic | `growing-conversation`, `large-tool-catalog`, `tool-call-burst`, `stage-transitions`, `classifier-mix` |+| Resilience | `context-overflow`, `failure-pressure`, `client-cancellation` |++## Check the local server and load tools++`scripts/run_local_soak_test.py` starts the crate's request-aware scenario backend and+`switchyard-server`, sends one HTTP request through each configured route, then runs `oha` and+NVIDIA AIPerf sequentially for every routing algorithm. The backend returns valid classifier+verdicts and injects the context, retry, stream, and cancellation cases named by resilience+scenarios. It resets transient-failure counters before each AIPerf cell so every routing algorithm+receives the same attempts. For streaming requests it emits the scenario's requested output length+with a configurable delay between tokens, which keeps local TTFT, ITL, and token-throughput+comparisons deterministic. The local test needs no provider key and incurs no inference cost.++Run `scripts/benchmark_routing_algorithms.py` by itself against an existing Switchyard server to+compare the same routes with real model output. oha runs only for the fixed `short-interactive`+baseline. AIPerf replays every selected session and reports TTFT, ITL, request throughput, output+token throughput, multi-run confidence, selected-target calls, classifier calls, and routing+overhead. Add `--direct-base-url` and `--direct-model` to compare each route with the same backend+without Switchyard. That comparison adds an annotated TTFT and output-token-throughput SVG to the+Markdown report. The operations guide explains how to keep the comparison fair and when to use the+local backend or real models.++The script needs the built Rust programs plus installed `oha` and AIPerf commands.+`switchyard-soak` itself does not require those extra programs and can run by itself against a+release server. See the [operations guide](../../docs/operations/soak_test.md) for setup, the local+command, the 48-hour release run, and result review.++## Results++Each run creates `config.json`, `intervals.csv`, `errors.jsonl`, and `summary.json`. Error details+stop at 10,000 records, and run-wide latency uses a 100,000-sample reservoir, so failures and a+high request rate cannot grow memory or disk use without bound.++See `docs/operations/soak_test.md` for release preparation, pass criteria, and result review.diff --git a/crates/switchyard-soak/examples/mock_server.rs b/crates/switchyard-soak/examples/mock_server.rsnew file mode 100644index 000000000..c92578fd6--- /dev/null+++ b/crates/switchyard-soak/examples/mock_server.rs@@ -0,0 +1,280 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Request-aware local backend used by `scripts/run_local_soak_test.py`.++use std::collections::HashMap;+use std::convert::Infallible;+use std::process::ExitCode;+use std::sync::Arc;+use std::time::Duration;++use axum::Router;+use axum::extract::{Json, State};+use axum::http::StatusCode;+use axum::response::sse::{Event, Sse};+use axum::response::{IntoResponse, Response};+use axum::routing::{get, post};+use clap::Parser;+use futures_util::{StreamExt, stream};+use parking_lot::Mutex;+use serde_json::{Value, json};++#[derive(Parser)]+#[command(+ name = "switchyard-soak-mock",+ about = "Start the request-aware backend for the local Switchyard scenario test",+ after_long_help = "Example:\n cargo run --release -p switchyard-soak --example switchyard-soak-mock -- --port 8100 --latency-ms 40",+ version+)]+struct Args {+ /// Local TCP port used by the backend.+ #[arg(long, default_value_t = 8100)]+ port: u16,++ /// Artificial delay, in milliseconds, added to ordinary responses.+ #[arg(long, default_value_t = 40)]+ latency_ms: u64,++ /// Artificial delay between streamed output tokens, in milliseconds.+ #[arg(long, default_value_t = 1)]+ token_latency_ms: u64,+}++#[derive(Clone)]+struct BackendState {+ latency: Duration,+ token_latency: Duration,+ attempts: Arc<Mutex<HashMap<String, u64>>>,+}++// AIPerf requires this OpenAI field; a fixed value keeps local runs deterministic.+const FIXED_CREATED_AT: u64 = 1_700_000_000;++fn scenario_marker(body: &Value) -> Option<&str> {+ body.get("messages")?+ .as_array()?+ .iter()+ .filter_map(|message| message.get("content").and_then(Value::as_str))+ .find_map(|content| {+ let marker = content.split_once("[scenario:")?.1;+ marker.split_once(']').map(|(name, _rest)| name)+ })+}++fn classifier_content(marker: Option<&str>) -> &'static str {+ if marker == Some("classifier_invalid") {+ return "not a JSON verdict";+ }+ if marker == Some("classifier_hard") {+ return r#"{"crux":"distributed diagnosis","primary_rule":"LIM-1","capability_boundary":"unsupported","p_solve":0.1}"#;+ }+ r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":1.0}"#+}++fn completion(model: &str, content: &str) -> Value {+ json!({+ "id": "chatcmpl-switchyard-soak",+ "object": "chat.completion",+ "created": FIXED_CREATED_AT,+ "model": model,+ "choices": [{+ "index": 0,+ "message": {"role": "assistant", "content": content},+ "finish_reason": "stop"+ }],+ "usage": {"prompt_tokens": 32, "completion_tokens": 2, "total_tokens": 34}+ })+}++fn requested_output_tokens(body: &Value) -> u64 {+ body.get("max_completion_tokens")+ .or_else(|| body.get("max_tokens"))+ .and_then(Value::as_u64)+ .unwrap_or(2)+ .clamp(1, 4_096)+}++fn stream(+ model: &str,+ truncated: bool,+ completion_tokens: u64,+ token_latency: Duration,+) -> Response {+ let mut events = Vec::with_capacity(completion_tokens as usize + 2);+ for index in 0..completion_tokens {+ events.push(+ json!({+ "id": "chatcmpl-switchyard-soak",+ "object": "chat.completion.chunk",+ "created": FIXED_CREATED_AT,+ "model": model,+ "choices": [{+ "index": 0,+ "delta": {+ "role": (index == 0).then_some("assistant"),+ "content": if index == 0 { "token" } else { " token" }+ },+ "finish_reason": null+ }]+ })+ .to_string(),+ );+ if truncated {+ break;+ }+ }+ if !truncated {+ events.push(+ json!({+ "id": "chatcmpl-switchyard-soak",+ "object": "chat.completion.chunk",+ "created": FIXED_CREATED_AT,+ "model": model,+ "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],+ "usage": {+ "prompt_tokens": 32,+ "completion_tokens": completion_tokens,+ "total_tokens": 32 + completion_tokens+ }+ })+ .to_string(),+ );+ events.push("[DONE]".to_string());+ }+ let events =+ stream::iter(events.into_iter().enumerate()).then(move |(index, data)| async move {+ if index > 0 {+ tokio::time::sleep(token_latency).await;+ }+ Ok::<Event, Infallible>(Event::default().data(data))+ });+ Sse::new(events).into_response()+}++async fn chat(State(state): State<BackendState>, Json(body): Json<Value>) -> Response {+ let model = body+ .get("model")+ .and_then(Value::as_str)+ .unwrap_or_default();+ let marker = scenario_marker(&body);+ if marker == Some("client_cancellation") {+ tokio::time::sleep(Duration::from_secs(2)).await;+ } else {+ tokio::time::sleep(state.latency).await;+ }++ if model == "mock/classifier" {+ return Json(completion(model, classifier_content(marker))).into_response();+ }+ if model == "mock/weak" && marker == Some("context_overflow") {+ return (+ StatusCode::BAD_REQUEST,+ Json(json!({+ "error": {+ "code": "context_length_exceeded",+ "message": "the weak target context window is too small"+ }+ })),+ )+ .into_response();+ }++ for (name, status) in [+ ("upstream_429", StatusCode::TOO_MANY_REQUESTS),+ ("upstream_500", StatusCode::INTERNAL_SERVER_ERROR),+ ] {+ if marker == Some(name) {+ let key = format!("{model}:{name}");+ let attempt = {+ let mut attempts = state.attempts.lock();+ let attempt = attempts.entry(key).or_default();+ *attempt += 1;+ *attempt+ };+ if attempt <= 2 {+ return (+ status,+ Json(+ json!({"error": {"message": format!("injected {name} attempt {attempt}")}}),+ ),+ )+ .into_response();+ }+ }+ }++ if body.get("stream") == Some(&Value::Bool(true)) {+ return stream(+ model,+ marker == Some("truncated_stream"),+ requested_output_tokens(&body),+ state.token_latency,+ );+ }+ Json(completion(model, "OK")).into_response()+}++async fn run(args: Args) -> Result<(), String> {+ if args.port == 0 {+ return Err("--port must be greater than zero".to_string());+ }+ let listener = tokio::net::TcpListener::bind(("127.0.0.1", args.port))+ .await+ .map_err(|error| error.to_string())?;+ let address = listener.local_addr().map_err(|error| error.to_string())?;+ let state = BackendState {+ latency: Duration::from_millis(args.latency_ms),+ token_latency: Duration::from_millis(args.token_latency_ms),+ attempts: Arc::new(Mutex::new(HashMap::new())),+ };+ let app = Router::new()+ .route("/health", get(|| async { Json(json!({"status": "ok"})) }))+ .route(+ "/reset",+ post(|State(state): State<BackendState>| async move {+ state.attempts.lock().clear();+ Json(json!({"status": "reset"}))+ }),+ )+ .route("/v1/chat/completions", post(chat))+ .with_state(state);+ println!("Scenario backend is ready at http://{address}");+ axum::serve(listener, app)+ .with_graceful_shutdown(async {+ if let Err(error) = tokio::signal::ctrl_c().await {+ eprintln!("could not register the shutdown signal: {error}");+ std::future::pending::<()>().await;+ }+ })+ .await+ .map_err(|error| error.to_string())+}++#[cfg(test)]+mod tests {+ use serde_json::json;++ use super::requested_output_tokens;++ #[test]+ fn output_tokens_follow_openai_limits_and_stay_bounded() {+ assert_eq!(requested_output_tokens(&json!({"max_tokens": 512})), 512);+ assert_eq!(+ requested_output_tokens(&json!({"max_completion_tokens": 8_192})),+ 4_096+ );+ assert_eq!(requested_output_tokens(&json!({})), 2);+ }+}++#[tokio::main]+async fn main() -> ExitCode {+ match run(Args::parse()).await {+ Ok(()) => ExitCode::SUCCESS,+ Err(error) => {+ eprintln!("switchyard-soak-mock failed: {error}");+ ExitCode::FAILURE+ }+ }+}diff --git a/crates/switchyard-soak/src/client.rs b/crates/switchyard-soak/src/client.rsnew file mode 100644index 000000000..5fe7074b1--- /dev/null+++ b/crates/switchyard-soak/src/client.rs@@ -0,0 +1,525 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! HTTP against the public Switchyard APIs: preflight, one request, and server-state reads.++use futures_util::StreamExt;+use reqwest::Client;+use reqwest::header::CONTENT_TYPE;+use serde_json::{Value, json};++const SERVER_REQUESTS_METRIC: &str = "switchyard_total_requests";+const SERVER_ERRORS_METRIC: &str = "switchyard_total_errors";++/// One public Switchyard API the soak test exercises.+#[derive(Clone, Copy, PartialEq, Eq)]+pub enum Endpoint {+ Chat,+ Messages,+ Responses,+}++impl Endpoint {+ pub const ALL: [Endpoint; 3] = [Endpoint::Chat, Endpoint::Messages, Endpoint::Responses];++ pub fn path(self) -> &'static str {+ match self {+ Endpoint::Chat => "/v1/chat/completions",+ Endpoint::Messages => "/v1/messages",+ Endpoint::Responses => "/v1/responses",+ }+ }++ /// Field a successful response for this endpoint must contain.+ fn required_field(self) -> &'static str {+ match self {+ Endpoint::Chat => "choices",+ Endpoint::Messages => "content",+ Endpoint::Responses => "output",+ }+ }++ pub fn as_str(self) -> &'static str {+ match self {+ Endpoint::Chat => "chat",+ Endpoint::Messages => "messages",+ Endpoint::Responses => "responses",+ }+ }+}++/// Keep at most *limit* characters, on a char boundary, for a logged detail string.+fn truncate(text: &str, limit: usize) -> String {+ text.chars().take(limit).collect()+}++fn transport_error_kind(error: &reqwest::Error) -> &'static str {+ if error.is_timeout() {+ "timeout"+ } else if error.is_decode() {+ "request_error"+ } else if error.is_connect() || error.is_request() || error.is_body() {+ "transport"+ } else {+ "request_error"+ }+}++/// Build one request body for a public Switchyard API.+pub fn request_body(+ endpoint: Endpoint,+ model: &str,+ prompt: &str,+ max_output_tokens: u32,+ stream: bool,+) -> Value {+ match endpoint {+ // Chat Completions and Anthropic Messages take the same model/messages/max_tokens body.+ Endpoint::Chat | Endpoint::Messages => json!({+ "model": model,+ "messages": [{"role": "user", "content": prompt}],+ "max_tokens": max_output_tokens,+ "temperature": 0,+ "stream": stream,+ }),+ Endpoint::Responses => json!({+ "model": model,+ "input": prompt,+ "max_output_tokens": max_output_tokens,+ "stream": stream,+ }),+ }+}++#[derive(Default)]+struct Metrics {+ requests: Option<f64>,+ errors: Option<f64>,+}++fn parse_metrics(text: &str) -> Metrics {+ let mut metrics = Metrics::default();+ for line in text.lines() {+ if line.is_empty() || line.starts_with('#') {+ continue;+ }+ let Some((name_part, rest)) = line.split_once(' ') else {+ continue;+ };+ let name = name_part.split('{').next().unwrap_or(name_part);+ if let Some(token) = rest.split_whitespace().next()+ && let Ok(value) = token.parse::<f64>()+ {+ match name {+ SERVER_REQUESTS_METRIC => metrics.requests = Some(value),+ SERVER_ERRORS_METRIC => metrics.errors = Some(value),+ _ => {}+ }+ }+ }+ metrics+}++#[derive(Debug)]+pub struct RequestError {+ pub kind: String,+ pub detail: String,+}++impl RequestError {+ fn new(kind: impl Into<String>, detail: impl Into<String>) -> Self {+ Self {+ kind: kind.into(),+ detail: detail.into(),+ }+ }++ fn transport(error: &reqwest::Error) -> Self {+ Self::new(+ transport_error_kind(error),+ truncate(&error.to_string(), 500),+ )+ }+}++struct StreamValidator {+ endpoint: Endpoint,+ event_name: Option<String>,+ saw_json: bool,+ saw_terminal: bool,+}++impl StreamValidator {+ fn new(endpoint: Endpoint) -> Self {+ Self {+ endpoint,+ event_name: None,+ saw_json: false,+ saw_terminal: false,+ }+ }++ fn read_line(&mut self, line: &[u8]) -> Result<(), RequestError> {+ let line = line.strip_suffix(b"\r").unwrap_or(line);+ if line.is_empty() {+ self.event_name = None;+ return Ok(());+ }+ let line = std::str::from_utf8(line)+ .map_err(|error| RequestError::new("invalid_stream", error.to_string()))?;+ if let Some(event_name) = line.strip_prefix("event:") {+ self.event_name = Some(event_name.trim_start().to_string());+ return Ok(());+ }+ let Some(data) = line.strip_prefix("data:") else {+ return Ok(());+ };+ let data = data.strip_prefix(' ').unwrap_or(data);+ if self.saw_terminal {+ return Err(RequestError::new(+ "invalid_stream",+ "stream contained data after its terminal event",+ ));+ }+ if data == "[DONE]" {+ if self.endpoint != Endpoint::Chat {+ return Err(RequestError::new(+ "invalid_stream",+ format!(+ "{} stream contained an OpenAI [DONE] marker",+ self.endpoint.as_str()+ ),+ ));+ }+ self.saw_terminal = true;+ return Ok(());+ }++ let payload: Value = serde_json::from_str(data).map_err(|error| {+ RequestError::new("invalid_stream", format!("invalid SSE JSON: {error}"))+ })?;+ let event_type = payload.get("type").and_then(Value::as_str);+ if self.event_name.as_deref() == Some("error")+ || event_type == Some("error")+ || payload.get("error").is_some()+ {+ let detail = payload+ .pointer("/error/message")+ .or_else(|| payload.get("message"))+ .and_then(Value::as_str)+ .unwrap_or("stream returned an error event");+ return Err(RequestError::new("stream_error", truncate(detail, 500)));+ }++ match self.endpoint {+ Endpoint::Chat => {}+ Endpoint::Messages | Endpoint::Responses => {+ let event_type = event_type.ok_or_else(|| {+ RequestError::new("invalid_stream", "SSE data did not contain an event type")+ })?;+ if self.event_name.as_deref() != Some(event_type) {+ return Err(RequestError::new(+ "invalid_stream",+ format!(+ "SSE event name {:?} did not match data type {event_type:?}",+ self.event_name+ ),+ ));+ }+ self.saw_terminal = matches!(+ (self.endpoint, event_type),+ (Endpoint::Messages, "message_stop")+ | (+ Endpoint::Responses,+ "response.completed" | "response.incomplete"+ )+ );+ }+ }+ self.saw_json = true;+ Ok(())+ }++ fn finish(self) -> Result<(), RequestError> {+ if !self.saw_json {+ return Err(RequestError::new(+ "empty_stream",+ "successful streaming response contained no JSON events",+ ));+ }+ if !self.saw_terminal {+ return Err(RequestError::new(+ "invalid_stream",+ format!(+ "{} stream ended without its terminal event",+ self.endpoint.as_str()+ ),+ ));+ }+ Ok(())+ }+}++/// Send one request and validate its response body or event stream.+pub async fn send_request(+ client: &Client,+ base_url: &str,+ endpoint: Endpoint,+ session_id: &str,+ body: &Value,+) -> Result<(), RequestError> {+ let url = format!("{base_url}{}", endpoint.path());+ let stream = body.get("stream").and_then(Value::as_bool).unwrap_or(false);+ let response = match client+ .post(&url)+ .header("x-switchyard-session-id", session_id)+ .json(body)+ .send()+ .await+ {+ Ok(response) => response,+ Err(error) => return Err(RequestError::transport(&error)),+ };++ if stream {+ let status = response.status();+ if !status.is_success() {+ let content = response.text().await.unwrap_or_default();+ return Err(RequestError::new(+ format!("http_{}", status.as_u16()),+ truncate(&content, 500),+ ));+ }+ let content_type = response+ .headers()+ .get(CONTENT_TYPE)+ .and_then(|value| value.to_str().ok())+ .unwrap_or("")+ .to_string();+ if !content_type.contains("text/event-stream") {+ let content = response.text().await.unwrap_or_default();+ return Err(RequestError::new(+ "invalid_stream",+ format!(+ "expected text/event-stream, received {content_type:?}: {}",+ truncate(&content, 300)+ ),+ ));+ }+ let mut bytes = response.bytes_stream();+ let mut pending = Vec::new();+ let mut validator = StreamValidator::new(endpoint);+ while let Some(chunk) = bytes.next().await {+ match chunk {+ Ok(chunk) => {+ pending.extend_from_slice(&chunk);+ while let Some(newline) = pending.iter().position(|byte| *byte == b'\n') {+ let mut line = pending.drain(..=newline).collect::<Vec<_>>();+ line.pop();+ validator.read_line(&line)?;+ }+ }+ Err(error) => {+ return Err(RequestError::transport(&error));+ }+ }+ }+ if !pending.is_empty() {+ validator.read_line(&pending)?;+ }+ return validator.finish();+ }++ let status = response.status();+ if !status.is_success() {+ let content = response.text().await.unwrap_or_default();+ return Err(RequestError::new(+ format!("http_{}", status.as_u16()),+ truncate(&content, 500),+ ));+ }+ let text = match response.text().await {+ Ok(text) => text,+ Err(error) => return Err(RequestError::transport(&error)),+ };+ let payload: Value = match serde_json::from_str(&text) {+ Ok(payload) => payload,+ Err(error) => return Err(RequestError::new("invalid_json", error.to_string())),+ };+ if !payload.is_object() || payload.get("error").is_some() {+ return Err(RequestError::new("invalid_response", truncate(&text, 500)));+ }+ let field = endpoint.required_field();+ if payload.get(field).is_none() {+ return Err(RequestError::new(+ "invalid_response",+ format!(+ "successful {} response did not contain {field:?}: {}",+ endpoint.as_str(),+ truncate(&text, 300)+ ),+ ));+ }+ Ok(())+}++/// Check liveness and verify that the requested model is advertised.+pub async fn preflight(client: &Client, base_url: &str, model: &str) -> Result<(), String> {+ let health = client+ .get(format!("{base_url}/health"))+ .send()+ .await+ .map_err(|error| error.to_string())?+ .error_for_status()+ .map_err(|error| error.to_string())?;+ let health_text = health.text().await.map_err(|error| error.to_string())?;+ let health_body: Value = serde_json::from_str(&health_text).map_err(|_| {+ format!(+ "GET /health did not return JSON: {}",+ truncate(&health_text, 300)+ )+ })?;+ if health_body.get("status").and_then(Value::as_str) != Some("ok") {+ return Err(format!(+ "GET /health returned an unexpected body: {}",+ truncate(&health_text, 300)+ ));+ }++ let response = client+ .get(format!("{base_url}/v1/models"))+ .send()+ .await+ .map_err(|error| error.to_string())?+ .error_for_status()+ .map_err(|error| error.to_string())?;+ let text = response.text().await.map_err(|error| error.to_string())?;+ let body: Value = serde_json::from_str(&text).map_err(|_| {+ format!(+ "GET /v1/models did not return JSON: {}",+ truncate(&text, 300)+ )+ })?;+ let entries = body.get("data").and_then(Value::as_array).ok_or_else(|| {+ format!(+ "GET /v1/models returned an unexpected body: {}",+ truncate(&text, 300)+ )+ })?;+ if !entries+ .iter()+ .any(|entry| entry.get("id").and_then(Value::as_str) == Some(model))+ {+ return Err(format!("model {model:?} is not listed by GET /v1/models"));+ }+ Ok(())+}++pub struct ServerState {+ pub healthy: bool,+ pub requests: Option<f64>,+ pub errors: Option<f64>,+}++/// Read liveness and cumulative metrics; one bad read becomes one failed sample.+pub async fn read_server_state(client: &Client, base_url: &str) -> ServerState {+ let healthy = match client.get(format!("{base_url}/health")).send().await {+ Ok(response) if response.status() == reqwest::StatusCode::OK => response+ .text()+ .await+ .ok()+ .and_then(|text| serde_json::from_str::<Value>(&text).ok())+ .and_then(|body| {+ body.get("status")+ .and_then(Value::as_str)+ .map(|s| s == "ok")+ })+ .unwrap_or(false),+ _ => false,+ };+ let metrics = match client.get(format!("{base_url}/metrics")).send().await {+ Ok(response) if response.status().is_success() => response+ .text()+ .await+ .map(|text| parse_metrics(&text))+ .unwrap_or_default(),+ _ => Metrics::default(),+ };+ ServerState {+ healthy,+ requests: metrics.requests,+ errors: metrics.errors,+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn request_bodies_match_public_endpoints() {+ let chat = request_body(Endpoint::Chat, "route", "hello", 8, true);+ let messages = request_body(Endpoint::Messages, "route", "hello", 8, true);+ let responses = request_body(Endpoint::Responses, "route", "hello", 8, true);++ assert_eq!(chat["messages"][0]["content"], json!("hello"));+ assert_eq!(chat["max_tokens"], json!(8));+ assert_eq!(messages["max_tokens"], json!(8));+ assert_eq!(responses["input"], json!("hello"));+ assert_eq!(responses["max_output_tokens"], json!(8));+ for body in [&chat, &messages, &responses] {+ assert_eq!(body["stream"], json!(true));+ }+ }++ #[test]+ fn parse_metrics_reads_only_required_counters() {+ let metrics = parse_metrics(+ "# TYPE switchyard_total_requests gauge\n\+ switchyard_total_requests 42\n\+ switchyard_total_errors{} 3\n\+ switchyard_requests_total{model=\"route\"} 10\n",+ );++ assert_eq!(metrics.requests, Some(42.0));+ assert_eq!(metrics.errors, Some(3.0));+ }++ #[test]+ fn stream_validator_accepts_each_public_terminal_event() -> Result<(), RequestError> {+ for (endpoint, stream) in [+ (Endpoint::Chat, "data: {\"choices\":[]}\n\ndata: [DONE]\n\n"),+ (+ Endpoint::Messages,+ "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n",+ ),+ (+ Endpoint::Responses,+ "event: response.completed\ndata: {\"type\":\"response.completed\"}\n\n",+ ),+ ] {+ let mut validator = StreamValidator::new(endpoint);+ for line in stream.split('\n') {+ validator.read_line(line.as_bytes())?;+ }+ validator.finish()?;+ }+ Ok(())+ }++ #[test]+ fn stream_validator_rejects_errors_and_missing_terminal_events() {+ let mut error = StreamValidator::new(Endpoint::Chat);+ let result = error.read_line(b"data: {\"error\":{\"message\":\"boom\"}}");+ assert_eq!(result.unwrap_err().kind, "stream_error");++ let mut incomplete = StreamValidator::new(Endpoint::Responses);+ incomplete+ .read_line(b"event: response.output_text.delta")+ .expect("event name should parse");+ incomplete+ .read_line(b"data: {\"type\":\"response.output_text.delta\",\"delta\":\"OK\"}")+ .expect("data should parse");+ assert_eq!(incomplete.finish().unwrap_err().kind, "invalid_stream");+ }+}diff --git a/crates/switchyard-soak/src/lib.rs b/crates/switchyard-soak/src/lib.rsnew file mode 100644index 000000000..779b70093--- /dev/null+++ b/crates/switchyard-soak/src/lib.rs@@ -0,0 +1,565 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++#![doc = include_str!("../README.md")]++mod client;+mod report;+mod scenarios;+mod stats;++use std::fs;+use std::future::Future;+use std::path::{Path, PathBuf};+use std::process::ExitCode;+use std::sync::Arc;+use std::sync::atomic::{AtomicBool, Ordering};+use std::time::{Duration, Instant};++use clap::Parser;+use parking_lot::Mutex;+use reqwest::Client;+use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue};+use serde_json::json;+use tokio::sync::Notify;++use crate::client::{Endpoint, preflight, send_request};+use crate::report::{ResultsWriter, invalid_request_canary, reporter};+use crate::scenarios::{Scenario, ScenarioOptions, ScenarioSet};+use crate::stats::{RunStats, build_summary, now_utc_string, round3, utc_dir_stamp};++/// Command-line arguments for the soak test.+#[derive(Parser)]+#[command(+ name = "switchyard-soak",+ about = "Run a sustained, closed-loop load test against a live Switchyard server",+ after_long_help = "Examples:\n switchyard-soak --model switchyard/general --duration 5m --concurrency 4\n switchyard-soak --model switchyard/general --scenario tool-call-burst --duration 5m\n switchyard-soak --model switchyard/general --export-scenarios scenarios\n\nThis command does not require oha or AIPerf. scripts/benchmark_routing_algorithms.py uses the exported scenario files with both tools.",+ version+)]+pub struct Args {+ /// HTTP base URL of the Switchyard server under test.+ #[arg(long, default_value = "http://127.0.0.1:4000")]+ base_url: String,++ /// Exact route id advertised by GET /v1/models.+ #[arg(long)]+ model: String,++ /// Time to keep sending load; use an s, m, or h suffix.+ #[arg(long, value_parser = stats::parse_duration, default_value = "48h")]+ duration: f64,++ /// Requests kept in flight; each worker sends its next request after the last one ends.+ #[arg(long, default_value_t = 16)]+ concurrency: usize,++ /// Default output-token limit; decode-heavy uses bounded 512 and 1,024 limits.+ #[arg(long, default_value_t = 32)]+ max_output_tokens: u32,++ /// Bytes of repeated prefix added to each prompt to exercise request memory and caching.+ #[arg(long, default_value_t = 1024)]+ prompt_bytes: usize,++ /// Named scenario collection; explicit --scenario values take precedence.+ #[arg(long, value_enum, default_value_t = ScenarioSet::Standard)]+ scenario_set: ScenarioSet,++ /// Exact scenario id to run; repeat to select more than one scenario.+ #[arg(long, action = clap::ArgAction::Append)]+ scenario: Vec<String>,++ /// Context-window size used to bound generated long-context scenarios.+ #[arg(long, default_value_t = 32_768)]+ context_window_tokens: usize,++ /// Export selected AIPerf input files and exit without contacting a server.+ #[arg(long)]+ export_scenarios: Option<PathBuf>,++ /// Seconds allowed to connect or wait between response bytes; an active stream may run longer.+ #[arg(long, default_value_t = 120.0)]+ request_timeout: f64,++ /// Seconds between health, metrics, process, and result samples.+ #[arg(long, default_value_t = 60.0)]+ report_interval: f64,++ /// Seconds between malformed-request checks; 0 turns this check off.+ #[arg(long, default_value_t = 300.0)]+ invalid_canary_interval: f64,++ /// Largest passing request-error fraction, from 0 (none) through 1 (all).+ #[arg(long, default_value_t = 0.0)]+ max_error_rate: f64,++ /// PID of a local switchyard-server process whose RSS and CPU should be sampled.+ #[arg(long)]+ server_pid: Option<u32>,++ /// Largest passing first-to-last RSS increase in MiB; requires --server-pid.+ #[arg(long, requires = "server_pid")]+ max_rss_growth_mib: Option<f64>,++ /// Name of an environment variable that holds the endpoint's bearer token.+ #[arg(long)]+ api_key_env: Option<String>,++ /// New directory to create for config, interval, error, and summary files.+ #[arg(long)]+ results_dir: Option<PathBuf>,+}++impl Args {+ /// Reject invalid numeric combinations after clap parses their types.+ pub fn validate(&self) -> Result<(), String> {+ if self.concurrency == 0 {+ return Err("--concurrency must be greater than zero".to_string());+ }+ if self.max_output_tokens == 0 || self.prompt_bytes == 0 {+ return Err(+ "--max-output-tokens and --prompt-bytes must be greater than zero".to_string(),+ );+ }+ ScenarioOptions {+ model: &self.model,+ prompt_bytes: self.prompt_bytes,+ max_output_tokens: self.max_output_tokens,+ context_window_tokens: self.context_window_tokens,+ }+ .validate()?;+ if !self.request_timeout.is_finite()+ || !self.report_interval.is_finite()+ || self.request_timeout <= 0.0+ || self.report_interval <= 0.0+ {+ return Err(+ "--request-timeout and --report-interval must be greater than zero".to_string(),+ );+ }+ if !self.invalid_canary_interval.is_finite() || self.invalid_canary_interval < 0.0 {+ return Err("--invalid-canary-interval must be zero or greater".to_string());+ }+ if !(0.0..=1.0).contains(&self.max_error_rate) {+ return Err("--max-error-rate must be between 0 and 1".to_string());+ }+ if self.server_pid == Some(0) {+ return Err("--server-pid must be greater than zero".to_string());+ }+ if self+ .max_rss_growth_mib+ .is_some_and(|growth| !growth.is_finite() || growth < 0.0)+ {+ return Err("--max-rss-growth-mib must be zero or greater".to_string());+ }+ Ok(())+ }+}++#[derive(Clone)]+struct RunContext {+ client: Client,+ base_url: String,+ stop: Arc<Stop>,+ stats: Arc<Mutex<RunStats>>,+ writer: Arc<Mutex<ResultsWriter>>,+}++struct Workload {+ model: String,+ scenarios: Vec<Scenario>,+}++/// A one-shot stop signal that many tasks can wait on and any task can raise.+struct Stop {+ flag: AtomicBool,+ notify: Notify,+}++impl Stop {+ fn new() -> Self {+ Self {+ flag: AtomicBool::new(false),+ notify: Notify::new(),+ }+ }++ fn set(&self) {+ self.flag.store(true, Ordering::SeqCst);+ self.notify.notify_waiters();+ }++ fn is_set(&self) -> bool {+ self.flag.load(Ordering::SeqCst)+ }++ /// Resolve once the signal is raised, now or later.+ async fn wait(&self) {+ loop {+ if self.is_set() {+ return;+ }+ // Register before the second flag check so a set() between the two still wakes us.+ let notified = self.notify.notified();+ tokio::pin!(notified);+ notified.as_mut().enable();+ if self.is_set() {+ return;+ }+ notified.await;+ }+ }+}++/// Send closed-loop traffic until the run stops.+async fn worker(+ context: RunContext,+ workload: Arc<Workload>,+ worker_id: usize,+) -> Result<(), String> {+ let mut request_number = 0;+ while !context.stop.is_set() {+ let scenario = &workload.scenarios[request_number % workload.scenarios.len()];+ let stream = (request_number / workload.scenarios.len()).is_multiple_of(2);+ let request = scenario.request(request_number / workload.scenarios.len(), stream);+ let session_id = format!("{}-worker-{worker_id}", request.session_id);+ let started = Instant::now();+ let result = send_request(+ &context.client,+ &context.base_url,+ request.endpoint,+ &session_id,+ request.body,+ )+ .await;+ let latency_ms = started.elapsed().as_secs_f64() * 1000.0;+ context.stats.lock().record(+ request.endpoint.as_str(),+ scenario.id,+ latency_ms,+ result.as_ref().err().map(|error| error.kind.as_str()),+ );+ if let Err(error) = result {+ context+ .writer+ .lock()+ .write_error(&json!({+ "timestamp_utc": now_utc_string(),+ "worker": worker_id,+ "scenario": scenario.id,+ "endpoint": request.endpoint.as_str(),+ "stream": stream,+ "latency_ms": round3(latency_ms),+ "error": error.kind,+ "detail": error.detail,+ }))+ .map_err(|error| error.to_string())?;+ }+ request_number = request_number.wrapping_add(1);+ }+ Ok(())+}++/// Stop the run when a background task fails or panics.+async fn guard_stop(+ stop: Arc<Stop>,+ task: impl Future<Output = Result<(), String>>,+) -> Result<(), String> {+ struct StopOnDrop(Arc<Stop>, bool);+ impl Drop for StopOnDrop {+ fn drop(&mut self) {+ if self.1 {+ self.0.set();+ }+ }+ }+ let mut guard = StopOnDrop(stop, true);+ let result = task.await;+ guard.1 = result.is_err();+ result+}++/// Add one joined task's error to the failure reasons; cancelled tasks are expected.+fn collect_failure(+ name: &str,+ result: Result<Result<(), String>, tokio::task::JoinError>,+ failures: &mut Vec<String>,+) {+ match result {+ Ok(Ok(())) => {}+ Ok(Err(reason)) => failures.push(format!("{name} failed: {reason}")),+ Err(join) if join.is_cancelled() => {}+ Err(join) => failures.push(format!("{name} failed: {join}")),+ }+}++/// Raise *stop* on SIGINT or SIGTERM so an operator can end the run cleanly.+fn spawn_signal_listener(stop: Arc<Stop>) -> tokio::task::JoinHandle<()> {+ tokio::spawn(async move {+ #[cfg(unix)]+ {+ use tokio::signal::unix::{SignalKind, signal};+ let mut interrupt = signal(SignalKind::interrupt()).ok();+ let mut terminate = signal(SignalKind::terminate()).ok();+ let wait_interrupt = async {+ match interrupt.as_mut() {+ Some(stream) => {+ stream.recv().await;+ }+ None => std::future::pending::<()>().await,+ }+ };+ let wait_terminate = async {+ match terminate.as_mut() {+ Some(stream) => {+ stream.recv().await;+ }+ None => std::future::pending::<()>().await,+ }+ };+ tokio::select! {+ _ = wait_interrupt => {}+ _ = wait_terminate => {}+ }+ stop.set();+ }+ #[cfg(not(unix))]+ {+ let _ = tokio::signal::ctrl_c().await;+ stop.set();+ }+ })+}++/// Record every non-secret input with the normalized duration and fixed request variants.+fn write_config(+ results_dir: &Path,+ args: &Args,+ model: &str,+ scenarios: &[Scenario],+) -> Result<(), String> {+ let config = json!({+ "base_url": args.base_url,+ "model": model,+ "duration_seconds": args.duration,+ "concurrency": args.concurrency,+ "endpoints": Endpoint::ALL.map(Endpoint::as_str),+ "streaming": [true, false],+ "max_output_tokens": args.max_output_tokens,+ "prompt_bytes": args.prompt_bytes,+ "context_window_tokens": args.context_window_tokens,+ "scenarios": scenarios.iter().map(|scenario| scenario.id).collect::<Vec<_>>(),+ "request_timeout": args.request_timeout,+ "report_interval": args.report_interval,+ "invalid_canary_interval": args.invalid_canary_interval,+ "max_error_rate": args.max_error_rate,+ "server_pid": args.server_pid,+ "max_rss_growth_mib": args.max_rss_growth_mib,+ "api_key_env": args.api_key_env,+ });+ let body = serde_json::to_string_pretty(&config).map_err(|error| error.to_string())?;+ fs::write(results_dir.join("config.json"), format!("{body}\n"))+ .map_err(|error| error.to_string())+}++/// Run the configured soak test and return a process exit code (0 pass, 1 fail).+pub async fn run(args: Args) -> Result<i32, String> {+ args.validate()?;++ let scenarios = scenarios::select(+ ScenarioOptions {+ model: &args.model,+ prompt_bytes: args.prompt_bytes,+ max_output_tokens: args.max_output_tokens,+ context_window_tokens: args.context_window_tokens,+ },+ args.scenario_set,+ &args.scenario,+ )?;+ if let Some(output_dir) = &args.export_scenarios {+ let manifest = scenarios::export(&scenarios, &args.model, output_dir)?;+ println!("Exported scenario manifest: {}", manifest.display());+ return Ok(0);+ }++ let token = match &args.api_key_env {+ Some(var) => {+ let token = std::env::var(var).ok().filter(|value| !value.is_empty());+ if token.is_none() {+ return Err(format!("${var} is not set"));+ }+ token+ }+ None => None,+ };++ // Per-operation timeouts, not a whole-request deadline: a healthy long stream that keeps+ // delivering bytes must not be aborted, so bound connect time and idle time between reads.+ let request_timeout = Duration::from_secs_f64(args.request_timeout);+ let mut builder = Client::builder()+ .no_proxy()+ .connect_timeout(request_timeout)+ .read_timeout(request_timeout)+ .pool_max_idle_per_host(args.concurrency + 4);+ if let Some(token) = &token {+ let mut headers = HeaderMap::new();+ let value = HeaderValue::from_str(&format!("Bearer {token}"))+ .map_err(|_| "the bearer token is not a valid HTTP header value".to_string())?;+ headers.insert(AUTHORIZATION, value);+ builder = builder.default_headers(headers);+ }+ let client = builder.build().map_err(|error| error.to_string())?;+ let base_url = args.base_url.trim_end_matches('/').to_string();++ preflight(&client, &base_url, &args.model).await?;+ let results_dir = args+ .results_dir+ .clone()+ .unwrap_or_else(|| PathBuf::from("soak-results").join(utc_dir_stamp()));+ let writer = Arc::new(Mutex::new(+ ResultsWriter::new(&results_dir).map_err(|error| error.to_string())?,+ ));+ write_config(&results_dir, &args, &args.model, &scenarios)?;++ println!(+ "Soak started: model={} duration={}s concurrency={} scenarios={} results={}",+ args.model,+ args.duration,+ args.concurrency,+ scenarios+ .iter()+ .map(|scenario| scenario.id)+ .collect::<Vec<_>>()+ .join(","),+ results_dir.display(),+ );++ let started = Instant::now();+ let stats = Arc::new(Mutex::new(RunStats::new(2026)));+ let stop = Arc::new(Stop::new());+ let workers_done = Arc::new(Stop::new());+ let context = RunContext {+ client,+ base_url,+ stop: stop.clone(),+ stats: stats.clone(),+ writer: writer.clone(),+ };++ let signal_listener = spawn_signal_listener(stop.clone());++ let deadline = {+ let stop = stop.clone();+ let stats = stats.clone();+ let duration = Duration::from_secs_f64(args.duration);+ tokio::spawn(async move {+ tokio::time::sleep(duration).await;+ stats.lock().completed_duration = true;+ stop.set();+ })+ };++ let workload = Arc::new(Workload {+ model: args.model.clone(),+ scenarios,+ });+ let mut worker_handles = Vec::new();+ for worker_id in 0..args.concurrency {+ let handle = tokio::spawn(guard_stop(+ stop.clone(),+ worker(context.clone(), workload.clone(), worker_id),+ ));+ worker_handles.push((format!("worker-{worker_id}"), handle));+ }+ let reporter_handle = tokio::spawn(guard_stop(+ stop.clone(),+ reporter(+ context.clone(),+ started,+ Duration::from_secs_f64(args.report_interval),+ args.duration,+ args.server_pid,+ workers_done.clone(),+ ),+ ));+ let canary_handle = tokio::spawn(guard_stop(+ stop.clone(),+ invalid_request_canary(context, args.invalid_canary_interval, workload),+ ));++ stop.wait().await;+ deadline.abort();+ signal_listener.abort();++ // A crashed worker/reporter/canary must not discard the run: record it as a failure reason+ // so the summary is still written and the run fails closed.+ let mut task_failures = Vec::new();+ for (name, handle) in worker_handles {+ collect_failure(&name, handle.await, &mut task_failures);+ }+ workers_done.set();+ collect_failure("reporter", reporter_handle.await, &mut task_failures);+ collect_failure(+ "invalid-request-canary",+ canary_handle.await,+ &mut task_failures,+ );++ let elapsed = started.elapsed().as_secs_f64();+ let (error_records, dropped_error_records) = { writer.lock().error_counts() };+ let summary = build_summary(+ &stats.lock(),+ elapsed,+ args.max_error_rate,+ args.max_rss_growth_mib,+ error_records,+ dropped_error_records,+ &task_failures,+ );+ let summary_path = results_dir.join("summary.json");+ let summary_body = serde_json::to_string_pretty(&summary).map_err(|error| error.to_string())?;+ fs::write(&summary_path, format!("{summary_body}\n")).map_err(|error| error.to_string())?;++ let label = if summary.passed { "PASS" } else { "FAIL" };+ println!(+ "Soak {label}: requests={} error_rate={:.4}% p95_ms={} summary={}",+ summary.requests,+ summary.error_rate * 100.0,+ summary+ .latency_p95_ms+ .map(|value| value.to_string())+ .unwrap_or_default(),+ summary_path.display(),+ );+ for reason in &summary.failure_reasons {+ println!("- {reason}");+ }+ Ok(if summary.passed { 0 } else { 1 })+}++/// Parse arguments, run the test on a multi-thread runtime, and map the result to an exit code.+pub fn cli_main() -> ExitCode {+ let args = Args::parse();+ if let Err(message) = args.validate() {+ eprintln!("switchyard-soak: {message}");+ return ExitCode::from(2);+ }+ let runtime = match tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()+ {+ Ok(runtime) => runtime,+ Err(error) => {+ eprintln!("soak test setup failed: {error}");+ return ExitCode::from(2);+ }+ };+ match runtime.block_on(run(args)) {+ Ok(0) => ExitCode::SUCCESS,+ Ok(_) => ExitCode::from(1),+ Err(error) => {+ eprintln!("soak test setup failed: {error}");+ ExitCode::from(2)+ }+ }+}diff --git a/crates/switchyard-soak/src/main.rs b/crates/switchyard-soak/src/main.rsnew file mode 100644index 000000000..8e3d676d5--- /dev/null+++ b/crates/switchyard-soak/src/main.rs@@ -0,0 +1,10 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Binary entrypoint for `switchyard-soak`.++use std::process::ExitCode;++fn main() -> ExitCode {+ switchyard_soak::cli_main()+}diff --git a/crates/switchyard-soak/src/report.rs b/crates/switchyard-soak/src/report.rsnew file mode 100644index 000000000..69e512528--- /dev/null+++ b/crates/switchyard-soak/src/report.rs@@ -0,0 +1,327 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Result files plus the background reporter and invalid-request canary tasks.++use std::fs::{self, File};+use std::io::{self, Write};+use std::path::Path;+use std::time::{Duration, Instant};++use serde_json::{Value, json};++use crate::client::read_server_state;+use crate::stats::{latency_stats, now_utc_string, round3};+use crate::{RunContext, Stop, Workload};++/// Cap on individually recorded failures so a bad run cannot fill the disk.+const MAX_ERROR_RECORDS: u64 = 10_000;++const INTERVAL_FIELDS: [&str; 15] = [+ "timestamp_utc",+ "elapsed_seconds",+ "requests",+ "successes",+ "failures",+ "requests_per_second",+ "latency_p50_ms",+ "latency_p95_ms",+ "latency_p99_ms",+ "latency_max_ms",+ "health",+ "server_total_requests",+ "server_total_errors",+ "rss_mib",+ "cpu_percent",+];++/// Write interval rows and bounded error details as the run proceeds.+pub struct ResultsWriter {+ interval_file: File,+ error_file: File,+ error_records: u64,+ dropped_error_records: u64,+}++impl ResultsWriter {+ /// Create a fresh results directory; fails if it already exists.+ pub fn new(results_dir: &Path) -> io::Result<Self> {+ if let Some(parent) = results_dir.parent()+ && !parent.as_os_str().is_empty()+ {+ fs::create_dir_all(parent)?;+ }+ fs::create_dir(results_dir)?;+ let mut interval_file = File::create(results_dir.join("intervals.csv"))?;+ writeln!(interval_file, "{}", INTERVAL_FIELDS.join(","))?;+ let error_file = File::create(results_dir.join("errors.jsonl"))?;+ Ok(Self {+ interval_file,+ error_file,+ error_records: 0,+ dropped_error_records: 0,+ })+ }++ /// Append one interval row; `cells` are already formatted in `INTERVAL_FIELDS` order.+ pub fn write_interval(&mut self, cells: &[String]) -> io::Result<()> {+ writeln!(self.interval_file, "{}", cells.join(","))?;+ self.interval_file.flush()+ }++ /// Append one failure record, dropping past the bound instead of growing without limit.+ pub fn write_error(&mut self, record: &Value) -> io::Result<()> {+ if self.error_records >= MAX_ERROR_RECORDS {+ self.dropped_error_records += 1;+ return Ok(());+ }+ writeln!(self.error_file, "{record}")?;+ self.error_file.flush()?;+ self.error_records += 1;+ Ok(())+ }++ pub fn error_counts(&self) -> (u64, u64) {+ (self.error_records, self.dropped_error_records)+ }+}++/// Format an optional number for a CSV cell; `None` becomes an empty cell.+fn cell(value: Option<f64>) -> String {+ value+ .map(|value| round3(value).to_string())+ .unwrap_or_default()+}++/// Return RSS MiB and CPU percent for *pid* using the local `ps` command.+pub async fn process_sample(pid: Option<u32>) -> (Option<f64>, Option<f64>) {+ let Some(pid) = pid else {+ return (None, None);+ };+ let output = match tokio::process::Command::new("ps")+ .args(["-o", "rss=,pcpu=", "-p", &pid.to_string()])+ .output()+ .await+ {+ Ok(output) => output,+ // ps missing or fork failed: record a process-check miss, don't crash the reporter.+ Err(_) => return (None, None),+ };+ if !output.status.success() {+ return (None, None);+ }+ let stdout = String::from_utf8_lossy(&output.stdout);+ let mut fields = stdout.split_whitespace();+ let (Some(rss), Some(cpu), None) = (fields.next(), fields.next(), fields.next()) else {+ return (None, None);+ };+ match (rss.parse::<f64>(), cpu.parse::<f64>()) {+ (Ok(rss_kib), Ok(cpu_percent)) => (Some(rss_kib / 1024.0), Some(cpu_percent)),+ _ => (None, None),+ }+}++/// Write one liveness, metrics, resource, and latency row per interval.+pub async fn reporter(+ context: RunContext,+ started: Instant,+ interval: Duration,+ target_seconds: f64,+ server_pid: Option<u32>,+ workers_done: std::sync::Arc<Stop>,+) -> Result<(), String> {+ let mut previous_report = started;+ loop {+ tokio::select! {+ _ = tokio::time::sleep(interval) => {}+ _ = context.stop.wait() => {}+ }+ // Once stopping, wait for the workers to drain so the final row counts their last requests.+ if context.stop.is_set() {+ workers_done.wait().await;+ }++ let now = Instant::now();+ let mut interval_stats = context.stats.lock().take_interval();+ let server = read_server_state(&context.client, &context.base_url).await;+ let (rss_mib, cpu_percent) = process_sample(server_pid).await;++ let (total_successes, total_failures) = {+ let mut state = context.stats.lock();+ state.health_checks += 1;+ state.metrics_checks += 1;+ if !server.healthy {+ state.health_failures += 1;+ }+ if server.requests.is_none() || server.errors.is_none() {+ state.metrics_failures += 1;+ }+ if server_pid.is_some() {+ state.process_checks += 1;+ if rss_mib.is_none() || cpu_percent.is_none() {+ state.process_failures += 1;+ }+ }+ if let Some(rss) = rss_mib {+ state.rss_samples.push(rss);+ }+ if let (Some(current), Some(previous)) =+ (server.requests, state.previous_server_requests)+ && current < previous+ {+ state.server_restarts += 1;+ }+ if let Some(current) = server.requests {+ state.previous_server_requests = Some(current);+ }+ (state.total_successes, state.total_failures)+ };++ let elapsed_interval = (now - previous_report).as_secs_f64().max(0.001);+ let requests = interval_stats.successes + interval_stats.failures;+ let latency = latency_stats(&mut interval_stats.latencies_ms);+ let requests_per_second = round3(requests as f64 / elapsed_interval);+ let timestamp = now_utc_string();+ let elapsed_seconds = round3((now - started).as_secs_f64());+ let health_label = if server.healthy { "ok" } else { "failed" };++ let cells = vec![+ timestamp.clone(),+ elapsed_seconds.to_string(),+ requests.to_string(),+ interval_stats.successes.to_string(),+ interval_stats.failures.to_string(),+ requests_per_second.to_string(),+ cell(latency.p50_ms),+ cell(latency.p95_ms),+ cell(latency.p99_ms),+ cell(latency.max_ms),+ health_label.to_string(),+ cell(server.requests),+ cell(server.errors),+ cell(rss_mib),+ cell(cpu_percent),+ ];+ // Cumulative, progress, and a glanceable status token so a remote tail of the log shows+ // at once that the run is alive, how far along it is, and whether it is healthy.+ let cumulative_requests = total_successes + total_failures;+ let cumulative_error_rate = if cumulative_requests > 0 {+ total_failures as f64 / cumulative_requests as f64+ } else {+ 0.0+ };+ let progress = (elapsed_seconds / target_seconds).min(1.0);+ let status = if requests == 0 {+ "stalled"+ } else if !server.healthy || interval_stats.failures > 0 {+ "degraded"+ } else {+ "ok"+ };+ let p95_text = latency.p95_ms.map(|v| v.to_string()).unwrap_or_default();+ let rss_text = rss_mib.map(|v| round3(v).to_string()).unwrap_or_default();++ context+ .writer+ .lock()+ .write_interval(&cells)+ .map_err(|error| error.to_string())?;++ println!(+ "[{timestamp}] progress={elapsed_seconds:.0}s/{target_seconds:.0}s({:.0}%) \+ reqs={cumulative_requests} interval={requests} errors={total_failures}({:.4}%) \+ rps={requests_per_second} p95_ms={p95_text} health={health_label} rss_mib={rss_text} \+ status={}",+ progress * 100.0,+ cumulative_error_rate * 100.0,+ status.to_uppercase(),+ );++ previous_report = now;+ if context.stop.is_set() {+ return Ok(());+ }+ }+}++/// Confirm invalid input returns 400 and the server stays live, on a fixed interval.+pub async fn invalid_request_canary(+ context: RunContext,+ interval: f64,+ workload: std::sync::Arc<Workload>,+) -> Result<(), String> {+ if interval <= 0.0 {+ return Ok(());+ }+ let interval = Duration::from_secs_f64(interval);+ loop {+ tokio::select! {+ _ = tokio::time::sleep(interval) => {}+ _ = context.stop.wait() => {}+ }+ if context.stop.is_set() {+ return Ok(());+ }++ context.stats.lock().canaries += 1;+ let probe = async {+ let invalid = context+ .client+ .post(format!("{}/v1/chat/completions", context.base_url))+ .json(&json!({"model": workload.model, "messages": "invalid"}))+ .send()+ .await?;+ let health = context+ .client+ .get(format!("{}/health", context.base_url))+ .send()+ .await?;+ Ok::<(u16, u16), reqwest::Error>((invalid.status().as_u16(), health.status().as_u16()))+ }+ .await;+ let (passed, detail) = match probe {+ Ok((invalid_status, health_status)) => (+ invalid_status == 400 && health_status == 200,+ format!("invalid_status={invalid_status}, health_status={health_status}"),+ ),+ Err(error) => (false, error.to_string()),+ };+ if !passed {+ context.stats.lock().canary_failures += 1;+ context+ .writer+ .lock()+ .write_error(&json!({+ "timestamp_utc": now_utc_string(),+ "error": "invalid_request_canary",+ "detail": detail,+ }))+ .map_err(|error| error.to_string())?;+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn error_details_stop_growing_at_the_file_limit() -> io::Result<()> {+ let parent = tempfile::tempdir()?;+ let results_dir = parent.path().join("results");+ let mut writer = ResultsWriter::new(&results_dir)?;++ for _ in 0..=MAX_ERROR_RECORDS {+ writer.write_error(&serde_json::json!({"error": "upstream"}))?;+ }++ assert_eq!(writer.error_counts(), (MAX_ERROR_RECORDS, 1));+ assert_eq!(+ fs::read_to_string(results_dir.join("errors.jsonl"))?+ .lines()+ .count(),+ 10_000+ );+ Ok(())+ }+}diff --git a/crates/switchyard-soak/src/scenarios/classifier_mix.rs b/crates/switchyard-soak/src/scenarios/classifier_mix.rsnew file mode 100644index 000000000..893366fdf--- /dev/null+++ b/crates/switchyard-soak/src/scenarios/classifier_mix.rs@@ -0,0 +1,42 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use super::{+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user,+};++pub fn build(options: ScenarioOptions<'_>) -> Scenario {+ let sessions = (0..20)+ .map(|index| {+ let hard = (8..10).contains(&index) || index >= 15;+ let marker = if hard {+ "classifier_hard"+ } else {+ "classifier_easy"+ };+ Session {+ session_id: format!("classifier-mix-{index}"),+ payloads: vec![chat_payload(+ options.model,+ vec![user(format!(+ "[scenario:{marker}] Request {index}: {}",+ if hard {+ "reason about a distributed failure with incomplete evidence"+ } else {+ "return the sum of two and two"+ }+ ))],+ options.max_output_tokens,+ )],+ }+ })+ .collect();+ Scenario::chat(+ "classifier-mix",+ ScenarioGroup::Agentic,+ "Two deterministic easy/hard mixes: 80/20 followed by 50/50.",+ "Selection shares match the input mix and classifier overhead is visible.",+ ErrorExpectation::SUCCESS,+ sessions,+ )+}diff --git a/crates/switchyard-soak/src/scenarios/client_cancellation.rs b/crates/switchyard-soak/src/scenarios/client_cancellation.rsnew file mode 100644index 000000000..66ccf29e3--- /dev/null+++ b/crates/switchyard-soak/src/scenarios/client_cancellation.rs@@ -0,0 +1,26 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use super::{+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user,+};++pub fn build(options: ScenarioOptions<'_>) -> Scenario {+ Scenario::chat(+ "client-cancellation",+ ScenarioGroup::Resilience,+ "Slow streaming requests whose client timeout cancels work in flight.",+ "Cancellation releases connections and does not destabilize later traffic.",+ ErrorExpectation::ALL,+ vec![Session {+ session_id: "client-cancellation".to_string(),+ payloads: vec![chat_payload(+ options.model,+ vec![user(+ "[scenario:client_cancellation] Delay this response until the client leaves.",+ )],+ options.max_output_tokens,+ )],+ }],+ )+}diff --git a/crates/switchyard-soak/src/scenarios/context_overflow.rs b/crates/switchyard-soak/src/scenarios/context_overflow.rsnew file mode 100644index 000000000..3e57e2007--- /dev/null+++ b/crates/switchyard-soak/src/scenarios/context_overflow.rs@@ -0,0 +1,26 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use super::{+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, token_text,+ user,+};++pub fn build(options: ScenarioOptions<'_>) -> Scenario {+ let tokens = (options.context_window_tokens * 9 / 10).max(1);+ Scenario::chat(+ "context-overflow",+ ScenarioGroup::Resilience,+ "A near-window request that one configured target rejects as too long.",+ "A multi-target route retries an eligible target and returns a valid response.",+ ErrorExpectation::SUCCESS,+ vec![Session {+ session_id: "context-overflow".to_string(),+ payloads: vec![chat_payload(+ options.model,+ vec![user(token_text(tokens, "context_overflow"))],+ options.max_output_tokens,+ )],+ }],+ )+}diff --git a/crates/switchyard-soak/src/scenarios/decode_heavy.rs b/crates/switchyard-soak/src/scenarios/decode_heavy.rsnew file mode 100644index 000000000..b19c4d68a--- /dev/null+++ b/crates/switchyard-soak/src/scenarios/decode_heavy.rs@@ -0,0 +1,30 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use super::{+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user,+};++pub fn build(options: ScenarioOptions<'_>) -> Scenario {+ let sessions = [512, 1024]+ .into_iter()+ .map(|tokens| Session {+ session_id: format!("decode-heavy-{tokens}"),+ payloads: vec![chat_payload(+ options.model,+ vec![user(format!(+ "[scenario:decode_heavy] Write {tokens} numbered one-word items."+ ))],+ tokens,+ )],+ })+ .collect();+ Scenario::chat(+ "decode-heavy",+ ScenarioGroup::Core,+ "Short inputs with 512-token and 1,024-token output limits.",+ "Output token throughput stays stable while long streams remain valid.",+ ErrorExpectation::SUCCESS,+ sessions,+ )+}diff --git a/crates/switchyard-soak/src/scenarios/failure_pressure.rs b/crates/switchyard-soak/src/scenarios/failure_pressure.rsnew file mode 100644index 000000000..a1606b065--- /dev/null+++ b/crates/switchyard-soak/src/scenarios/failure_pressure.rs@@ -0,0 +1,36 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use super::{+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user,+};++pub fn build(options: ScenarioOptions<'_>) -> Scenario {+ let cases = [+ ("truncated-stream", "truncated_stream"),+ ("upstream-429", "upstream_429"),+ ("upstream-500", "upstream_500"),+ ("classifier-invalid", "classifier_invalid"),+ ];+ let sessions = cases+ .into_iter()+ .map(|(id, marker)| Session {+ session_id: id.to_string(),+ payloads: vec![chat_payload(+ options.model,+ vec![user(format!(+ "[scenario:{marker}] Exercise the configured recovery path."+ ))],+ options.max_output_tokens,+ )],+ })+ .collect();+ Scenario::chat(+ "failure-pressure",+ ScenarioGroup::Resilience,+ "Bounded 429, 500, malformed-classifier, and truncated-stream injections.",+ "Retries recover transient failures; terminal failures remain explicit and bounded.",+ ErrorExpectation::MIXED,+ sessions,+ )+}diff --git a/crates/switchyard-soak/src/scenarios/growing_conversation.rs b/crates/switchyard-soak/src/scenarios/growing_conversation.rsnew file mode 100644index 000000000..17da931aa--- /dev/null+++ b/crates/switchyard-soak/src/scenarios/growing_conversation.rs@@ -0,0 +1,34 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use super::{+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, assistant, chat_payload,+ user,+};++pub fn build(options: ScenarioOptions<'_>) -> Scenario {+ let mut messages = Vec::new();+ let mut payloads = Vec::new();+ for turn in 0..8 {+ messages.push(user(format!(+ "[scenario:growing_conversation] Turn {turn}: remember the number {turn}."+ )));+ payloads.push(chat_payload(+ options.model,+ messages.clone(),+ options.max_output_tokens.min(64),+ ));+ messages.push(assistant(format!("I recorded {turn}.")));+ }+ Scenario::chat(+ "growing-conversation",+ ScenarioGroup::Agentic,+ "An eight-turn session whose complete message history grows each turn.",+ "Session affinity holds and latency tracks the growing history.",+ ErrorExpectation::SUCCESS,+ vec![Session {+ session_id: "growing-conversation".to_string(),+ payloads,+ }],+ )+}diff --git a/crates/switchyard-soak/src/scenarios/large_tool_catalog.rs b/crates/switchyard-soak/src/scenarios/large_tool_catalog.rsnew file mode 100644index 000000000..a95363f18--- /dev/null+++ b/crates/switchyard-soak/src/scenarios/large_tool_catalog.rs@@ -0,0 +1,61 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use serde_json::{Value, json};++use super::{+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user,+};++fn tools(count: usize) -> Vec<Value> {+ (0..count)+ .map(|index| {+ json!({+ "type": "function",+ "function": {+ "name": format!("lookup_record_{index}"),+ "description": format!("Look up record {index} in the test catalog."),+ "parameters": {+ "type": "object",+ "properties": {+ "query": {"type": "string"},+ "limit": {"type": "integer", "minimum": 1, "maximum": 10}+ },+ "required": ["query"],+ "additionalProperties": false+ }+ }+ })+ })+ .collect()+}++pub fn build(options: ScenarioOptions<'_>) -> Scenario {+ let sessions = [16, 64]+ .into_iter()+ .map(|count| {+ let mut payload = chat_payload(+ options.model,+ vec![user(format!(+ "[scenario:large_tool_catalog] Use lookup_record_{} for query green.",+ count - 1+ ))],+ options.max_output_tokens.min(128),+ );+ payload["tools"] = Value::Array(tools(count));+ payload["tool_choice"] = json!("auto");+ Session {+ session_id: format!("large-tool-catalog-{count}"),+ payloads: vec![payload],+ }+ })+ .collect();+ Scenario::chat(+ "large-tool-catalog",+ ScenarioGroup::Agentic,+ "Requests with 16-tool and 64-tool JSON-schema catalogs.",+ "Routing overhead remains bounded while large tool schemas are forwarded intact.",+ ErrorExpectation::SUCCESS,+ sessions,+ )+}diff --git a/crates/switchyard-soak/src/scenarios/long_context.rs b/crates/switchyard-soak/src/scenarios/long_context.rsnew file mode 100644index 000000000..6958e847f--- /dev/null+++ b/crates/switchyard-soak/src/scenarios/long_context.rs@@ -0,0 +1,36 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use super::{+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, token_text,+ user,+};++pub fn build(options: ScenarioOptions<'_>) -> Scenario {+ let mut lengths = vec![8_192, 32_000, options.context_window_tokens * 9 / 10];+ lengths.retain(|length| *length < options.context_window_tokens);+ lengths.sort_unstable();+ lengths.dedup();+ let sessions = lengths+ .into_iter()+ .map(|tokens| Session {+ session_id: format!("long-context-{tokens}"),+ payloads: vec![chat_payload(+ options.model,+ vec![user(format!(+ "{}\nSummarize the final marker in one sentence.",+ token_text(tokens, "long_context")+ ))],+ options.max_output_tokens.min(64),+ )],+ })+ .collect();+ Scenario::chat(+ "long-context",+ ScenarioGroup::Core,+ "Requests at 8K, 32K, and near the configured context window.",+ "TTFT rises with input length without routing errors or memory growth.",+ ErrorExpectation::SUCCESS,+ sessions,+ )+}diff --git a/crates/switchyard-soak/src/scenarios/mixed_traffic.rs b/crates/switchyard-soak/src/scenarios/mixed_traffic.rsnew file mode 100644index 000000000..ae1a444a4--- /dev/null+++ b/crates/switchyard-soak/src/scenarios/mixed_traffic.rs@@ -0,0 +1,41 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use super::{+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, token_text,+ user,+};++pub fn build(options: ScenarioOptions<'_>) -> Scenario {+ let lengths = [16, 24, 32, 48, 64, 96, 128, 1_024, 2_048, 8_192];+ let sessions = lengths+ .into_iter()+ .enumerate()+ .map(|(index, tokens)| Session {+ session_id: format!("mixed-{index}"),+ payloads: vec![chat_payload(+ options.model,+ vec![user(format!(+ "{}\nReply with the request number {index}.",+ token_text(+ tokens.min(options.context_window_tokens / 2),+ "mixed_traffic"+ )+ ))],+ if index == 9 {+ 256+ } else {+ options.max_output_tokens+ },+ )],+ })+ .collect();+ Scenario::chat(+ "mixed-traffic",+ ScenarioGroup::Core,+ "A 70/20/10 mix of short, medium, and long requests.",+ "Tail latency stays bounded when unlike request sizes share one route.",+ ErrorExpectation::SUCCESS,+ sessions,+ )+}diff --git a/crates/switchyard-soak/src/scenarios/mod.rs b/crates/switchyard-soak/src/scenarios/mod.rsnew file mode 100644index 000000000..b7bf9a2cf--- /dev/null+++ b/crates/switchyard-soak/src/scenarios/mod.rs@@ -0,0 +1,528 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Reproducible request scenarios shared by the soak command and AIPerf runner.++mod classifier_mix;+mod client_cancellation;+mod context_overflow;+mod decode_heavy;+mod failure_pressure;+mod growing_conversation;+mod large_tool_catalog;+mod long_context;+mod mixed_traffic;+mod prefix_reuse;+mod short_interactive;+mod stage_transitions;+mod tool_call_burst;++use std::collections::HashSet;+use std::fs;+use std::path::{Path, PathBuf};++use clap::ValueEnum;+use serde::Serialize;+use serde_json::{Value, json};++use crate::client::Endpoint;++/// Largest accepted context-window input for generated scenario data.+const MAX_CONTEXT_WINDOW_TOKENS: usize = 1_000_000;+const MIN_CONTEXT_WINDOW_TOKENS: usize = 16_384;+const MAX_PROMPT_BYTES: usize = 1_000_000;++/// Scenario family used by CLI selection and report grouping.+#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]+#[serde(rename_all = "snake_case")]+pub enum ScenarioGroup {+ Core,+ Agentic,+ Resilience,+}++/// Accepted client-visible error-rate range for one scenario.+#[derive(Clone, Copy, Serialize)]+pub struct ErrorExpectation {+ pub min_rate: f64,+ pub max_rate: f64,+}++impl ErrorExpectation {+ pub const SUCCESS: Self = Self {+ min_rate: 0.0,+ max_rate: 0.0,+ };+ pub const MIXED: Self = Self {+ min_rate: 0.01,+ max_rate: 0.75,+ };+ pub const ALL: Self = Self {+ min_rate: 0.8,+ max_rate: 1.0,+ };+}++/// Named scenario collection selected by the soak and benchmark commands.+#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)]+#[value(rename_all = "snake_case")]+pub enum ScenarioSet {+ Core,+ Agentic,+ Resilience,+ Standard,+ All,+}++impl ScenarioSet {+ fn contains(self, group: ScenarioGroup) -> bool {+ match self {+ Self::Core => group == ScenarioGroup::Core,+ Self::Agentic => group == ScenarioGroup::Agentic,+ Self::Resilience => group == ScenarioGroup::Resilience,+ Self::Standard => group != ScenarioGroup::Resilience,+ Self::All => true,+ }+ }+}++/// Request-generation limits shared by every scenario builder.+#[derive(Clone, Copy)]+pub struct ScenarioOptions<'a> {+ pub model: &'a str,+ pub prompt_bytes: usize,+ pub max_output_tokens: u32,+ pub context_window_tokens: usize,+}++impl ScenarioOptions<'_> {+ pub fn validate(&self) -> Result<(), String> {+ if self.model.is_empty() {+ return Err("scenario model must not be empty".to_string());+ }+ if self.prompt_bytes == 0 || self.max_output_tokens == 0 {+ return Err("scenario prompt and output limits must be greater than zero".to_string());+ }+ if !(MIN_CONTEXT_WINDOW_TOKENS..=MAX_CONTEXT_WINDOW_TOKENS)+ .contains(&self.context_window_tokens)+ {+ return Err(format!(+ "--context-window-tokens must be between {MIN_CONTEXT_WINDOW_TOKENS} and {MAX_CONTEXT_WINDOW_TOKENS}"+ ));+ }+ let prompt_limit = MAX_PROMPT_BYTES.min(self.context_window_tokens.saturating_mul(2));+ if self.prompt_bytes > prompt_limit {+ return Err(format!(+ "--prompt-bytes must not exceed {prompt_limit} for this context window"+ ));+ }+ Ok(())+ }+}++/// One AIPerf conversation; payloads remain ordered within the session.+#[derive(Clone, Serialize)]+pub struct Session {+ pub session_id: String,+ pub payloads: Vec<Value>,+}++/// One request the Rust soak command can send.+pub struct RequestCase {+ pub endpoint: Endpoint,+ pub session_id: String,+ streaming_body: Value,+ nonstreaming_body: Value,+}++impl RequestCase {+ pub fn new(endpoint: Endpoint, session_id: String, streaming_body: Value) -> Self {+ let mut nonstreaming_body = streaming_body.clone();+ nonstreaming_body["stream"] = Value::Bool(false);+ Self {+ endpoint,+ session_id,+ streaming_body,+ nonstreaming_body,+ }+ }+}++pub struct PreparedRequest<'a> {+ pub endpoint: Endpoint,+ pub session_id: &'a str,+ pub body: &'a Value,+}++/// QPS point expressed relative to the benchmark command's base request rate.+#[derive(Clone, Serialize)]+pub struct RatePoint {+ pub time_s: u32,+ pub rate_multiplier: f64,+}++/// AIPerf schedule applied to a request scenario.+#[derive(Clone, Serialize)]+#[serde(tag = "kind", rename_all = "snake_case")]+pub enum LoadProfile {+ Fixed {+ id: &'static str,+ },+ ConcurrencyKnee {+ id: &'static str,+ concurrency_steps: Vec<usize>,+ },+ TrafficBurst {+ id: &'static str,+ duration_seconds: u32,+ points: Vec<RatePoint>,+ },+}++/// Immutable scenario data built once before load workers start.+pub struct Scenario {+ pub id: &'static str,+ pub group: ScenarioGroup,+ pub description: &'static str,+ pub expected: &'static str,+ pub expected_error_rate: ErrorExpectation,+ pub sessions: Vec<Session>,+ pub soak_requests: Vec<RequestCase>,+ pub load_profiles: Vec<LoadProfile>,+}++impl Scenario {+ /// Build a Chat Completions scenario whose AIPerf payloads also drive soak traffic.+ pub fn chat(+ id: &'static str,+ group: ScenarioGroup,+ description: &'static str,+ expected: &'static str,+ expected_error_rate: ErrorExpectation,+ sessions: Vec<Session>,+ ) -> Self {+ let soak_requests =+ sessions+ .iter()+ .flat_map(|session| {+ session.payloads.iter().cloned().map(|body| {+ RequestCase::new(Endpoint::Chat, session.session_id.clone(), body)+ })+ })+ .collect();+ Self {+ id,+ group,+ description,+ expected,+ expected_error_rate,+ sessions,+ soak_requests,+ load_profiles: fixed_load(),+ }+ }++ /// Return one prebuilt request with only its streaming flag changed.+ pub fn request(&self, index: usize, stream: bool) -> PreparedRequest<'_> {+ let selected = &self.soak_requests[index % self.soak_requests.len()];+ PreparedRequest {+ endpoint: selected.endpoint,+ session_id: &selected.session_id,+ body: if stream {+ &selected.streaming_body+ } else {+ &selected.nonstreaming_body+ },+ }+ }+}++/// Return the one fixed schedule used by ordinary request scenarios.+pub fn fixed_load() -> Vec<LoadProfile> {+ vec![LoadProfile::Fixed { id: "fixed" }]+}++/// Build the three bounded load schedules used for the short baseline.+pub fn baseline_load() -> Vec<LoadProfile> {+ vec![+ LoadProfile::Fixed { id: "fixed" },+ LoadProfile::ConcurrencyKnee {+ id: "concurrency-knee",+ concurrency_steps: vec![1, 4, 16, 64, 128],+ },+ LoadProfile::TrafficBurst {+ id: "traffic-burst",+ duration_seconds: 30,+ points: vec![+ RatePoint {+ time_s: 0,+ rate_multiplier: 1.0,+ },+ RatePoint {+ time_s: 10,+ rate_multiplier: 1.0,+ },+ RatePoint {+ time_s: 11,+ rate_multiplier: 10.0,+ },+ RatePoint {+ time_s: 16,+ rate_multiplier: 10.0,+ },+ RatePoint {+ time_s: 17,+ rate_multiplier: 1.0,+ },+ RatePoint {+ time_s: 30,+ rate_multiplier: 1.0,+ },+ ],+ },+ ]+}++/// Build one raw Chat Completions payload.+pub fn chat_payload(model: &str, messages: Vec<Value>, max_tokens: u32) -> Value {+ json!({+ "model": model,+ "messages": messages,+ "max_tokens": max_tokens,+ "temperature": 0,+ "stream": true,+ })+}++/// Build one user message.+pub fn user(content: impl Into<String>) -> Value {+ json!({"role": "user", "content": content.into()})+}++/// Build one assistant message.+pub fn assistant(content: impl Into<String>) -> Value {+ json!({"role": "assistant", "content": content.into()})+}++/// Build repeated context with a stable approximate token count.+pub fn token_text(tokens: usize, label: &str) -> String {+ format!("[scenario:{label}] {}", "x ".repeat(tokens))+}++/// Build the complete catalog in stable report order.+pub fn catalog(options: ScenarioOptions<'_>) -> Result<Vec<Scenario>, String> {+ options.validate()?;+ Ok(vec![+ short_interactive::build(options),+ long_context::build(options),+ decode_heavy::build(options),+ prefix_reuse::build(options),+ mixed_traffic::build(options),+ growing_conversation::build(options),+ large_tool_catalog::build(options),+ tool_call_burst::build(options),+ stage_transitions::build(options),+ classifier_mix::build(options),+ context_overflow::build(options),+ failure_pressure::build(options),+ client_cancellation::build(options),+ ])+}++/// Select explicit scenario IDs or every scenario in a named set.+pub fn select(+ options: ScenarioOptions<'_>,+ set: ScenarioSet,+ requested: &[String],+) -> Result<Vec<Scenario>, String> {+ let mut catalog = catalog(options)?;+ if requested.is_empty() {+ return Ok(catalog+ .into_iter()+ .filter(|scenario| set.contains(scenario.group))+ .collect());+ }++ let mut seen = HashSet::new();+ let mut selected = Vec::new();+ for id in requested {+ if !seen.insert(id) {+ return Err(format!("scenario {id:?} was requested more than once"));+ }+ let index = catalog+ .iter()+ .position(|scenario| scenario.id == id)+ .ok_or_else(|| format!("unknown scenario {id:?}"))?;+ selected.push(catalog.remove(index));+ }+ Ok(selected)+}++#[derive(Serialize)]+struct InputsFile<'a> {+ data: &'a [Session],+}++#[derive(Serialize)]+struct Manifest<'a> {+ schema_version: u32,+ model: &'a str,+ scenarios: Vec<ManifestScenario<'a>>,+}++#[derive(Serialize)]+struct ManifestScenario<'a> {+ id: &'a str,+ group: ScenarioGroup,+ description: &'a str,+ expected: &'a str,+ expected_error_rate: ErrorExpectation,+ input_file: String,+ load_profiles: &'a [LoadProfile],+}++/// Export the selected scenario catalog in AIPerf's verbatim inputs JSON format.+pub fn export(scenarios: &[Scenario], model: &str, output_dir: &Path) -> Result<PathBuf, String> {+ if output_dir.exists() {+ return Err(format!(+ "scenario output directory already exists: {}",+ output_dir.display()+ ));+ }+ let inputs_dir = output_dir.join("inputs");+ fs::create_dir_all(&inputs_dir).map_err(|error| error.to_string())?;++ let mut entries = Vec::new();+ for scenario in scenarios {+ let relative = format!("inputs/{}.json", scenario.id);+ let path = output_dir.join(&relative);+ let body = serde_json::to_string_pretty(&InputsFile {+ data: &scenario.sessions,+ })+ .map_err(|error| error.to_string())?;+ fs::write(path, format!("{body}\n")).map_err(|error| error.to_string())?;+ entries.push(ManifestScenario {+ id: scenario.id,+ group: scenario.group,+ description: scenario.description,+ expected: scenario.expected,+ expected_error_rate: scenario.expected_error_rate,+ input_file: relative,+ load_profiles: &scenario.load_profiles,+ });+ }++ let manifest_path = output_dir.join("manifest.json");+ let body = serde_json::to_string_pretty(&Manifest {+ schema_version: 1,+ model,+ scenarios: entries,+ })+ .map_err(|error| error.to_string())?;+ fs::write(&manifest_path, format!("{body}\n")).map_err(|error| error.to_string())?;+ Ok(manifest_path)+}++#[cfg(test)]+mod tests {+ use super::*;++ fn options() -> ScenarioOptions<'static> {+ ScenarioOptions {+ model: "switchyard/test",+ prompt_bytes: 128,+ max_output_tokens: 32,+ context_window_tokens: 16_384,+ }+ }++ #[test]+ fn catalog_is_unique_bounded_and_serializable() -> Result<(), String> {+ let scenarios = catalog(options())?;+ let names = scenarios+ .iter()+ .map(|scenario| scenario.id)+ .collect::<HashSet<_>>();+ assert_eq!(names.len(), scenarios.len());+ assert!(scenarios.iter().all(|scenario| {+ !scenario.sessions.is_empty()+ && !scenario.soak_requests.is_empty()+ && scenario.sessions.len() <= 32+ && scenario+ .sessions+ .iter()+ .all(|session| !session.payloads.is_empty() && session.payloads.len() <= 32)+ }));++ let parent = tempfile::tempdir().map_err(|error| error.to_string())?;+ let output = parent.path().join("export");+ let manifest = export(&scenarios, options().model, &output)?;+ let payload: Value =+ serde_json::from_str(&fs::read_to_string(manifest).map_err(|error| error.to_string())?)+ .map_err(|error| error.to_string())?;+ assert_eq!(payload["scenarios"].as_array().map(Vec::len), Some(13));+ Ok(())+ }++ #[test]+ fn explicit_selection_keeps_order_and_rejects_bad_names() -> Result<(), String> {+ let requested = vec!["tool-call-burst".to_string(), "long-context".to_string()];+ let selected = select(options(), ScenarioSet::Core, &requested)?;+ assert_eq!(+ selected+ .iter()+ .map(|scenario| scenario.id)+ .collect::<Vec<_>>(),+ ["tool-call-burst", "long-context"]+ );+ assert!(select(options(), ScenarioSet::Core, &["missing".to_string()]).is_err());+ Ok(())+ }++ #[test]+ fn agentic_payloads_keep_their_bounded_contracts() -> Result<(), String> {+ let scenarios = catalog(options())?;+ let find = |id| {+ scenarios+ .iter()+ .find(|scenario| scenario.id == id)+ .ok_or_else(|| format!("missing scenario {id}"))+ };++ let long = find("long-context")?;+ assert!(long.sessions.iter().any(|session| {+ session.payloads[0]["messages"][0]["content"]+ .as_str()+ .is_some_and(|content| content.len() > 8_000)+ }));++ let tools = find("large-tool-catalog")?;+ assert_eq!(+ tools.sessions[1].payloads[0]["tools"]+ .as_array()+ .map(Vec::len),+ Some(64)+ );++ let burst = find("tool-call-burst")?;+ assert_eq!(burst.sessions[0].payloads.len(), 9);+ let final_messages = burst.sessions[0].payloads[8]["messages"]+ .as_array()+ .ok_or_else(|| "tool-call burst messages were not an array".to_string())?;+ assert!(+ final_messages.iter().any(|message| {+ message["role"] == "tool" && message["tool_call_id"] == "call_7"+ })+ );++ let growing = find("growing-conversation")?;+ let message_counts = growing.sessions[0]+ .payloads+ .iter()+ .map(|payload| payload["messages"].as_array().map(Vec::len).unwrap_or(0))+ .collect::<Vec<_>>();+ assert!(message_counts.windows(2).all(|pair| pair[0] < pair[1]));+ assert!(find("failure-pressure")?.expected_error_rate.min_rate > 0.0);+ Ok(())+ }+}diff --git a/crates/switchyard-soak/src/scenarios/prefix_reuse.rs b/crates/switchyard-soak/src/scenarios/prefix_reuse.rsnew file mode 100644index 000000000..7272ec501--- /dev/null+++ b/crates/switchyard-soak/src/scenarios/prefix_reuse.rs@@ -0,0 +1,37 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use super::{+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, token_text,+ user,+};++pub fn build(options: ScenarioOptions<'_>) -> Scenario {+ let prefix_tokens = (options.prompt_bytes / 2).clamp(512, 8_192);+ let shared = token_text(prefix_tokens, "prefix_reuse_shared");+ let sessions = (0..8)+ .map(|index| {+ let prefix = if index < 4 {+ shared.clone()+ } else {+ token_text(prefix_tokens, &format!("prefix_reuse_unique_{index}"))+ };+ Session {+ session_id: format!("prefix-reuse-{index}"),+ payloads: vec![chat_payload(+ options.model,+ vec![user(format!("{prefix}\nReturn item {index}."))],+ options.max_output_tokens.min(32),+ )],+ }+ })+ .collect();+ Scenario::chat(+ "prefix-reuse",+ ScenarioGroup::Core,+ "Matched requests with shared and unique long prefixes.",+ "The report exposes cache-sensitive TTFT without changing route behavior.",+ ErrorExpectation::SUCCESS,+ sessions,+ )+}diff --git a/crates/switchyard-soak/src/scenarios/short_interactive.rs b/crates/switchyard-soak/src/scenarios/short_interactive.rsnew file mode 100644index 000000000..023f04161--- /dev/null+++ b/crates/switchyard-soak/src/scenarios/short_interactive.rs@@ -0,0 +1,61 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use crate::client::{Endpoint, request_body};++use super::{+ ErrorExpectation, RequestCase, Scenario, ScenarioGroup, ScenarioOptions, Session,+ baseline_load, chat_payload, user,+};++pub fn build(options: ScenarioOptions<'_>) -> Scenario {+ let prompts = [+ "Reply with exactly OK.".to_string(),+ "Name one primary color.".to_string(),+ "Return the number after 41.".to_string(),+ format!(+ "[scenario:short_interactive] {} Reply briefly.",+ "stable prefix ".repeat(options.prompt_bytes.div_ceil(14))+ ),+ ];+ let sessions = prompts+ .iter()+ .enumerate()+ .map(|(index, prompt)| Session {+ session_id: format!("short-{index}"),+ payloads: vec![chat_payload(+ options.model,+ vec![user(prompt)],+ options.max_output_tokens,+ )],+ })+ .collect();+ let soak_requests = Endpoint::ALL+ .iter()+ .flat_map(|endpoint| {+ prompts.iter().enumerate().map(move |(index, prompt)| {+ RequestCase::new(+ *endpoint,+ format!("short-{index}-{}", endpoint.as_str()),+ request_body(+ *endpoint,+ options.model,+ prompt,+ options.max_output_tokens,+ true,+ ),+ )+ })+ })+ .collect();+ Scenario {+ id: "short-interactive",+ group: ScenarioGroup::Core,+ description: "Short prompts that establish HTTP, routing, TTFT, and latency overhead.",+ expected: "All public endpoints succeed; oha and AIPerf establish the baseline.",+ expected_error_rate: ErrorExpectation::SUCCESS,+ sessions,+ soak_requests,+ load_profiles: baseline_load(),+ }+}diff --git a/crates/switchyard-soak/src/scenarios/stage_transitions.rs b/crates/switchyard-soak/src/scenarios/stage_transitions.rsnew file mode 100644index 000000000..63916e37e--- /dev/null+++ b/crates/switchyard-soak/src/scenarios/stage_transitions.rs@@ -0,0 +1,53 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use serde_json::json;++use super::{+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user,+};++pub fn build(options: ScenarioOptions<'_>) -> Scenario {+ let mut messages = vec![user(+ "[scenario:stage_exploration] Find the failing module, then fix it.",+ )];+ let mut payloads = vec![chat_payload(+ options.model,+ messages.clone(),+ options.max_output_tokens.min(128),+ )];++ messages.extend([+ json!({"role":"assistant","tool_calls":[{"id":"stage_1","type":"function","function":{"name":"run_tests","arguments":"{}"}}]}),+ json!({"role":"tool","tool_call_id":"stage_1","content":"fatal: critical tool failure: repository index is unavailable"}),+ ]);+ payloads.push(chat_payload(+ options.model,+ messages.clone(),+ options.max_output_tokens.min(128),+ ));++ messages.extend([+ json!({"role":"assistant","tool_calls":[{"id":"stage_2","type":"function","function":{"name":"apply_patch","arguments":"{}"}}]}),+ json!({"role":"tool","tool_call_id":"stage_2","content":"patch applied"}),+ json!({"role":"assistant","tool_calls":[{"id":"stage_3","type":"function","function":{"name":"run_tests","arguments":"{}"}}]}),+ json!({"role":"tool","tool_call_id":"stage_3","content":"all tests passed"}),+ ]);+ payloads.push(chat_payload(+ options.model,+ messages,+ options.max_output_tokens.min(128),+ ));++ Scenario::chat(+ "stage-transitions",+ ScenarioGroup::Agentic,+ "One growing session that moves through exploration, critical failure, and productive work.",+ "The stage router changes tiers at the configured evidence boundaries.",+ ErrorExpectation::SUCCESS,+ vec![Session {+ session_id: "stage-transitions".to_string(),+ payloads,+ }],+ )+}diff --git a/crates/switchyard-soak/src/scenarios/tool_call_burst.rs b/crates/switchyard-soak/src/scenarios/tool_call_burst.rsnew file mode 100644index 000000000..1c6b35d69--- /dev/null+++ b/crates/switchyard-soak/src/scenarios/tool_call_burst.rs@@ -0,0 +1,65 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use serde_json::{Value, json};++use super::{+ ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user,+};++pub fn build(options: ScenarioOptions<'_>) -> Scenario {+ let mut messages = vec![user(+ "[scenario:tool_call_burst] Inspect eight shards, one call at a time.",+ )];+ let mut payloads = vec![chat_payload(+ options.model,+ messages.clone(),+ options.max_output_tokens.min(128),+ )];+ for index in 0..8 {+ let call_id = format!("call_{index}");+ messages.push(json!({+ "role": "assistant",+ "content": null,+ "tool_calls": [{+ "id": call_id,+ "type": "function",+ "function": {"name": "inspect_shard", "arguments": format!("{{\"shard\":{index}}}")}+ }]+ }));+ messages.push(json!({+ "role": "tool",+ "tool_call_id": format!("call_{index}"),+ "content": format!("shard {index}: healthy")+ }));+ payloads.push(chat_payload(+ options.model,+ messages.clone(),+ options.max_output_tokens.min(128),+ ));+ }+ for payload in &mut payloads {+ payload["tools"] = Value::Array(vec![json!({+ "type": "function",+ "function": {+ "name": "inspect_shard",+ "parameters": {+ "type": "object",+ "properties": {"shard": {"type": "integer"}},+ "required": ["shard"]+ }+ }+ })]);+ }+ Scenario::chat(+ "tool-call-burst",+ ScenarioGroup::Agentic,+ "An eight-turn burst of linked assistant tool calls and tool results.",+ "The route keeps session state and forwards every call/result pair.",+ ErrorExpectation::SUCCESS,+ vec![Session {+ session_id: "tool-call-burst".to_string(),+ payloads,+ }],+ )+}diff --git a/crates/switchyard-soak/src/stats.rs b/crates/switchyard-soak/src/stats.rsnew file mode 100644index 000000000..dcb8f1f20--- /dev/null+++ b/crates/switchyard-soak/src/stats.rs@@ -0,0 +1,453 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Bounded in-memory run state, latency percentiles, and the final pass/fail summary.++use std::collections::BTreeMap;+use std::time::SystemTime;++use rand::rngs::StdRng;+use rand::{RngExt, SeedableRng};+use serde::Serialize;++/// Cap on retained latency samples; a long run stays within this bound by reservoir sampling.+const RESERVOIR_SIZE: usize = 100_000;++/// Round to three decimals so result files stay readable.+pub fn round3(value: f64) -> f64 {+ (value * 1000.0).round() / 1000.0+}++/// Parse a duration such as `30s`, `15m`, or `48h` into seconds.+pub fn parse_duration(value: &str) -> Result<f64, String> {+ let text = value.trim().to_lowercase();+ let bad = || "duration must use s, m, or h, for example 30s or 48h".to_string();+ let unit = text.chars().last().ok_or_else(bad)?;+ let multiplier = match unit {+ 's' => 1.0,+ 'm' => 60.0,+ 'h' => 3600.0,+ _ => return Err(bad()),+ };+ let number = &text[..text.len() - unit.len_utf8()];+ let seconds = number.parse::<f64>().map_err(|_| bad())? * multiplier;+ if !seconds.is_finite() || seconds <= 0.0 {+ return Err("duration must be greater than zero".to_string());+ }+ Ok(seconds)+}++#[derive(Default)]+pub struct LatencyStats {+ pub p50_ms: Option<f64>,+ pub p95_ms: Option<f64>,+ pub p99_ms: Option<f64>,+ pub max_ms: Option<f64>,+}++fn percentile_sorted(values: &[f64], quantile: f64) -> Option<f64> {+ let index = (((values.len().checked_sub(1)?) as f64) * quantile).round_ties_even() as usize;+ Some(round3(values[index]))+}++pub fn latency_stats(values: &mut [f64]) -> LatencyStats {+ values.sort_by(|a, b| a.total_cmp(b));+ LatencyStats {+ p50_ms: percentile_sorted(values, 0.50),+ p95_ms: percentile_sorted(values, 0.95),+ p99_ms: percentile_sorted(values, 0.99),+ max_ms: values.last().copied().map(round3),+ }+}++/// ISO-8601 UTC timestamp at second precision, e.g. `2026-07-30T18:04:00Z`.+pub fn now_utc_string() -> String {+ humantime::format_rfc3339_seconds(SystemTime::now()).to_string()+}++/// Compact UTC stamp for a results directory, e.g. `20260730T180400Z`.+pub fn utc_dir_stamp() -> String {+ now_utc_string()+ .chars()+ .filter(|character| !matches!(character, '-' | ':'))+ .collect()+}++/// Request results collected since the previous report.+#[derive(Default)]+pub struct IntervalStats {+ pub successes: u64,+ pub failures: u64,+ pub latencies_ms: Vec<f64>,+}++/// Bounded in-memory state for one soak run.+pub struct RunStats {+ rng: StdRng,+ interval: IntervalStats,+ pub total_successes: u64,+ pub total_failures: u64,+ pub endpoint_successes: BTreeMap<String, u64>,+ pub endpoint_failures: BTreeMap<String, u64>,+ pub scenario_successes: BTreeMap<String, u64>,+ pub scenario_failures: BTreeMap<String, u64>,+ pub error_kinds: BTreeMap<String, u64>,+ latency_reservoir: Vec<f64>,+ latency_count: u64,+ pub health_checks: u64,+ pub health_failures: u64,+ pub metrics_checks: u64,+ pub metrics_failures: u64,+ pub process_checks: u64,+ pub process_failures: u64,+ pub canaries: u64,+ pub canary_failures: u64,+ pub server_restarts: u64,+ pub previous_server_requests: Option<f64>,+ pub rss_samples: Vec<f64>,+ pub completed_duration: bool,+}++impl RunStats {+ pub fn new(seed: u64) -> Self {+ Self {+ rng: StdRng::seed_from_u64(seed),+ interval: IntervalStats::default(),+ total_successes: 0,+ total_failures: 0,+ endpoint_successes: BTreeMap::new(),+ endpoint_failures: BTreeMap::new(),+ scenario_successes: BTreeMap::new(),+ scenario_failures: BTreeMap::new(),+ error_kinds: BTreeMap::new(),+ latency_reservoir: Vec::new(),+ latency_count: 0,+ health_checks: 0,+ health_failures: 0,+ metrics_checks: 0,+ metrics_failures: 0,+ process_checks: 0,+ process_failures: 0,+ canaries: 0,+ canary_failures: 0,+ server_restarts: 0,+ previous_server_requests: None,+ rss_samples: Vec::new(),+ completed_duration: false,+ }+ }++ /// Record one completed inference request; `error_kind` is `None` on success.+ pub fn record(+ &mut self,+ endpoint: &str,+ scenario: &str,+ latency_ms: f64,+ error_kind: Option<&str>,+ ) {+ match error_kind {+ None => {+ self.interval.successes += 1;+ self.total_successes += 1;+ *self+ .endpoint_successes+ .entry(endpoint.to_string())+ .or_default() += 1;+ *self+ .scenario_successes+ .entry(scenario.to_string())+ .or_default() += 1;+ }+ Some(kind) => {+ self.interval.failures += 1;+ self.total_failures += 1;+ *self+ .endpoint_failures+ .entry(endpoint.to_string())+ .or_default() += 1;+ *self+ .scenario_failures+ .entry(scenario.to_string())+ .or_default() += 1;+ *self.error_kinds.entry(kind.to_string()).or_default() += 1;+ }+ }+ self.interval.latencies_ms.push(latency_ms);+ self.latency_count += 1;+ if self.latency_reservoir.len() < RESERVOIR_SIZE {+ self.latency_reservoir.push(latency_ms);+ } else {+ let index = self.rng.random_range(0..self.latency_count) as usize;+ if index < RESERVOIR_SIZE {+ self.latency_reservoir[index] = latency_ms;+ }+ }+ }++ /// Return and reset the current interval.+ pub fn take_interval(&mut self) -> IntervalStats {+ std::mem::take(&mut self.interval)+ }+}++/// Final result written to `summary.json`.+#[derive(Serialize)]+pub struct Summary {+ pub passed: bool,+ pub failure_reasons: Vec<String>,+ pub completed_duration: bool,+ pub elapsed_seconds: f64,+ pub requests: u64,+ pub successes: u64,+ pub failures: u64,+ pub error_rate: f64,+ pub requests_per_second: f64,+ pub endpoint_successes: BTreeMap<String, u64>,+ pub endpoint_failures: BTreeMap<String, u64>,+ pub scenario_successes: BTreeMap<String, u64>,+ pub scenario_failures: BTreeMap<String, u64>,+ pub error_kinds: BTreeMap<String, u64>,+ pub latency_p50_ms: Option<f64>,+ pub latency_p95_ms: Option<f64>,+ pub latency_p99_ms: Option<f64>,+ pub health_checks: u64,+ pub health_failures: u64,+ pub metrics_checks: u64,+ pub metrics_failures: u64,+ pub process_checks: u64,+ pub process_failures: u64,+ pub invalid_request_canaries: u64,+ pub invalid_request_canary_failures: u64,+ pub detected_server_restarts: u64,+ pub rss_first_mib: Option<f64>,+ pub rss_last_mib: Option<f64>,+ pub rss_max_mib: Option<f64>,+ pub rss_growth_mib: Option<f64>,+ pub error_records: u64,+ pub dropped_error_records: u64,+}++/// Build the final result and its release-gate reasons.+pub fn build_summary(+ stats: &RunStats,+ elapsed_seconds: f64,+ max_error_rate: f64,+ max_rss_growth_mib: Option<f64>,+ error_records: u64,+ dropped_error_records: u64,+ task_failures: &[String],+) -> Summary {+ let total = stats.total_successes + stats.total_failures;+ let error_rate = if total > 0 {+ stats.total_failures as f64 / total as f64+ } else {+ 1.0+ };+ let rss_first = stats.rss_samples.first().copied();+ let rss_last = stats.rss_samples.last().copied();+ let rss_growth = match (rss_first, rss_last) {+ (Some(first), Some(last)) => Some(last - first),+ _ => None,+ };++ let mut reasons: Vec<String> = task_failures.to_vec();+ if !stats.completed_duration {+ reasons.push("the run stopped before the requested duration".to_string());+ }+ if total == 0 {+ reasons.push("no inference requests completed".to_string());+ }+ if error_rate > max_error_rate {+ reasons.push(format!(+ "request error rate {:.4}% exceeded the {:.4}% limit",+ error_rate * 100.0,+ max_error_rate * 100.0+ ));+ }+ if stats.health_failures > 0 {+ reasons.push(format!("{} liveness checks failed", stats.health_failures));+ }+ if stats.metrics_failures > 0 {+ reasons.push(format!(+ "{} server metrics checks failed",+ stats.metrics_failures+ ));+ }+ if stats.process_failures > 0 {+ reasons.push(format!(+ "{} server process checks failed",+ stats.process_failures+ ));+ }+ if stats.canary_failures > 0 {+ reasons.push(format!(+ "{} invalid-request recovery checks failed",+ stats.canary_failures+ ));+ }+ if stats.server_restarts > 0 {+ reasons.push(format!(+ "server counters reset {} time(s)",+ stats.server_restarts+ ));+ }+ if let (Some(limit), Some(growth)) = (max_rss_growth_mib, rss_growth)+ && growth > limit+ {+ reasons.push(format!(+ "server RSS grew {growth:.1} MiB, above the {limit:.1} MiB limit"+ ));+ }++ let rss_max = stats+ .rss_samples+ .iter()+ .copied()+ .fold(None, |acc: Option<f64>, value| {+ Some(acc.map_or(value, |m| m.max(value)))+ });+ let requests_per_second = if elapsed_seconds > 0.0 {+ total as f64 / elapsed_seconds+ } else {+ 0.0+ };++ let mut latency_samples = stats.latency_reservoir.clone();+ let latency = latency_stats(&mut latency_samples);++ Summary {+ passed: reasons.is_empty(),+ failure_reasons: reasons,+ completed_duration: stats.completed_duration,+ elapsed_seconds: round3(elapsed_seconds),+ requests: total,+ successes: stats.total_successes,+ failures: stats.total_failures,+ error_rate,+ requests_per_second,+ endpoint_successes: stats.endpoint_successes.clone(),+ endpoint_failures: stats.endpoint_failures.clone(),+ scenario_successes: stats.scenario_successes.clone(),+ scenario_failures: stats.scenario_failures.clone(),+ error_kinds: stats.error_kinds.clone(),+ latency_p50_ms: latency.p50_ms,+ latency_p95_ms: latency.p95_ms,+ latency_p99_ms: latency.p99_ms,+ health_checks: stats.health_checks,+ health_failures: stats.health_failures,+ metrics_checks: stats.metrics_checks,+ metrics_failures: stats.metrics_failures,+ process_checks: stats.process_checks,+ process_failures: stats.process_failures,+ invalid_request_canaries: stats.canaries,+ invalid_request_canary_failures: stats.canary_failures,+ detected_server_restarts: stats.server_restarts,+ rss_first_mib: rss_first,+ rss_last_mib: rss_last,+ rss_max_mib: rss_max,+ rss_growth_mib: rss_growth,+ error_records,+ dropped_error_records,+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn parse_duration_accepts_units_and_rejects_invalid_values() {+ assert_eq!(parse_duration("30s"), Ok(30.0));+ assert_eq!(parse_duration(".5s"), Ok(0.5));+ assert_eq!(parse_duration("2.5m"), Ok(150.0));+ assert_eq!(parse_duration("48h"), Ok(172_800.0));+ for value in ["0s", "10", "5x", "-3s", "1d", "NaNs", "infs"] {+ assert!(parse_duration(value).is_err(), "{value} should be rejected");+ }+ }++ #[test]+ fn latency_stats_sorts_once_and_handles_empty_input() {+ let mut values = [60.0, 10.0, 50.0, 20.0, 40.0, 30.0];+ let latency = latency_stats(&mut values);+ assert_eq!(latency.p50_ms, Some(30.0));+ assert_eq!(latency.p95_ms, Some(60.0));+ assert_eq!(latency.p99_ms, Some(60.0));+ assert_eq!(latency.max_ms, Some(60.0));++ let empty = latency_stats(&mut []);+ assert_eq!(empty.p50_ms, None);+ assert_eq!(empty.max_ms, None);+ }++ #[test]+ fn record_tracks_interval_and_cumulative_results() {+ let mut stats = RunStats::new(1);+ stats.record("chat", "short-interactive", 10.0, None);+ stats.record("messages", "long-context", 20.0, Some("timeout"));++ let interval = stats.take_interval();+ assert_eq!(interval.successes, 1);+ assert_eq!(interval.failures, 1);+ assert_eq!(stats.total_successes, 1);+ assert_eq!(stats.total_failures, 1);+ assert_eq!(stats.error_kinds.get("timeout"), Some(&1));+ }++ #[test]+ fn latency_samples_stay_bounded_after_the_reservoir_fills() {+ let mut stats = RunStats::new(1);+ for sample in 0..=RESERVOIR_SIZE {+ stats.record("chat", "short-interactive", sample as f64, None);+ }++ assert_eq!(stats.latency_reservoir.len(), RESERVOIR_SIZE);+ assert_eq!(stats.latency_count, RESERVOIR_SIZE as u64 + 1);+ }++ #[test]+ fn summary_reports_all_failed_gates() {+ let mut stats = RunStats::new(1);+ stats.total_successes = 998;+ stats.total_failures = 2;+ stats.health_failures = 1;+ stats.metrics_failures = 1;+ stats.process_failures = 1;+ stats.canary_failures = 1;+ stats.server_restarts = 1;+ stats.rss_samples = vec![100.0, 700.0];++ let summary = build_summary(+ &stats,+ 10.0,+ 0.001,+ Some(512.0),+ 0,+ 0,+ &["reporter failed: boom".to_string()],+ );++ assert!(!summary.passed);+ for reason in [+ "run stopped",+ "error rate",+ "liveness",+ "metrics",+ "process",+ "invalid-request",+ "counters reset",+ "RSS grew",+ "reporter failed",+ ] {+ assert!(+ summary+ .failure_reasons+ .iter()+ .any(|message| message.contains(reason)),+ "missing {reason:?}: {:?}",+ summary.failure_reasons,+ );+ }+ assert_eq!(summary.rss_growth_mib, Some(600.0));+ }+}diff --git a/crates/switchyard-soak/tests/soak.rs b/crates/switchyard-soak/tests/soak.rsnew file mode 100644index 000000000..26c8dcbda--- /dev/null+++ b/crates/switchyard-soak/tests/soak.rs@@ -0,0 +1,331 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! End-to-end tests against a local server that implements the public Switchyard APIs.++use std::collections::HashSet;+use std::error::Error;+use std::sync::Arc;++use axum::Router;+use axum::extract::{Json, State};+use axum::http::{HeaderMap, StatusCode, header};+use axum::response::{IntoResponse, Response};+use axum::routing::{get, post};+use clap::Parser;+use parking_lot::Mutex;+use serde_json::{Value, json};++use switchyard_soak::{Args, run};++type TestResult = Result<(), Box<dyn Error>>;++struct TestServer {+ base_url: String,+ task: tokio::task::JoinHandle<()>,+}++impl Drop for TestServer {+ fn drop(&mut self) {+ self.task.abort();+ }+}++async fn serve(app: Router) -> Result<TestServer, Box<dyn Error>> {+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;+ let addr = listener.local_addr()?;+ let task = tokio::spawn(async move {+ let _ = axum::serve(listener, app.into_make_service()).await;+ });+ Ok(TestServer {+ base_url: format!("http://{addr}"),+ task,+ })+}++#[derive(Clone, Default)]+struct MockState {+ is_chat_broken: bool,+ seen: Arc<Mutex<HashSet<String>>>,+}++impl MockState {+ fn record(&self, endpoint: &str, body: &Value) {+ let stream = body.get("stream").and_then(Value::as_bool) == Some(true);+ self.seen.lock().insert(format!("{endpoint}:{stream}"));+ }+}++fn switchyard(is_chat_broken: bool) -> (Router, Arc<Mutex<HashSet<String>>>) {+ async fn health() -> Response {+ Json(json!({"status": "ok"})).into_response()+ }+ async fn metrics() -> Response {+ "switchyard_total_requests 1\nswitchyard_total_errors 0\n".into_response()+ }+ async fn models() -> Response {+ Json(json!({"data": [{"id": "soak-route"}]})).into_response()+ }+ async fn chat(+ State(state): State<MockState>,+ headers: HeaderMap,+ Json(body): Json<Value>,+ ) -> Response {+ let invalid_messages = body.get("messages").is_some_and(|messages| {+ messages+ .as_array()+ .is_none_or(|messages| messages.is_empty())+ });+ if invalid_messages {+ return if state.is_chat_broken {+ Json(json!({"choices": []})).into_response()+ } else {+ (StatusCode::BAD_REQUEST, "messages must not be empty").into_response()+ };+ }+ if !headers.contains_key("x-switchyard-session-id") {+ return (StatusCode::BAD_REQUEST, "missing session id").into_response();+ }+ state.record("chat", &body);+ if state.is_chat_broken {+ return if body.get("stream") == Some(&Value::Bool(true)) {+ Json(json!({"choices": []})).into_response()+ } else {+ Json(json!({})).into_response()+ };+ }+ response(&body, "choices")+ }+ async fn messages(+ State(state): State<MockState>,+ headers: HeaderMap,+ Json(body): Json<Value>,+ ) -> Response {+ if !headers.contains_key("x-switchyard-session-id") {+ return (StatusCode::BAD_REQUEST, "missing session id").into_response();+ }+ state.record("messages", &body);+ response(&body, "content")+ }+ async fn responses(+ State(state): State<MockState>,+ headers: HeaderMap,+ Json(body): Json<Value>,+ ) -> Response {+ if !headers.contains_key("x-switchyard-session-id") {+ return (StatusCode::BAD_REQUEST, "missing session id").into_response();+ }+ state.record("responses", &body);+ response(&body, "output")+ }+ fn response(body: &Value, field: &str) -> Response {+ if body.get("stream") == Some(&Value::Bool(true)) {+ let stream = match field {+ "choices" => "data: {\"choices\":[]}\n\ndata: [DONE]\n\n",+ "content" => "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n",+ "output" => {+ "event: response.completed\ndata: {\"type\":\"response.completed\"}\n\n"+ }+ _ => "",+ };+ ([(header::CONTENT_TYPE, "text/event-stream")], stream).into_response()+ } else {+ let mut response = serde_json::Map::new();+ response.insert(field.to_string(), json!([]));+ Json(Value::Object(response)).into_response()+ }+ }++ let state = MockState {+ is_chat_broken,+ ..MockState::default()+ };+ let seen = state.seen.clone();+ let app = Router::new()+ .route("/health", get(health))+ .route("/metrics", get(metrics))+ .route("/v1/models", get(models))+ .route("/v1/chat/completions", post(chat))+ .route("/v1/messages", post(messages))+ .route("/v1/responses", post(responses))+ .with_state(state);+ (app, seen)+}++fn args(base_url: &str, results_dir: &str) -> Result<Args, clap::Error> {+ Args::try_parse_from([+ "switchyard-soak",+ "--base-url",+ base_url,+ "--model",+ "soak-route",+ "--duration",+ "0.8s",+ "--concurrency",+ "2",+ "--report-interval",+ "0.1",+ "--invalid-canary-interval",+ "0.1",+ "--scenario",+ "short-interactive",+ "--results-dir",+ results_dir,+ ])+}++#[tokio::test]+async fn short_run_exercises_every_response_variant_and_passes() -> TestResult {+ let (app, seen) = switchyard(false);+ let server = serve(app).await?;+ let dir = tempfile::tempdir()?;+ let results_dir = dir.path().join("soak-results");+ let args = args(+ &server.base_url,+ results_dir.to_str().ok_or("non-utf8 results dir")?,+ )?;++ args.validate()?;+ assert_eq!(run(args).await?, 0);++ let summary: Value =+ serde_json::from_str(&std::fs::read_to_string(results_dir.join("summary.json"))?)?;+ assert_eq!(summary["passed"], json!(true));+ assert!(summary["invalid_request_canaries"].as_u64().unwrap_or(0) > 0);+ assert_eq!(+ *seen.lock(),+ HashSet::from([+ "chat:false".to_string(),+ "chat:true".to_string(),+ "messages:false".to_string(),+ "messages:true".to_string(),+ "responses:false".to_string(),+ "responses:true".to_string(),+ ])+ );+ Ok(())+}++#[tokio::test]+async fn bad_responses_and_canary_write_a_failing_summary() -> TestResult {+ let (app, _) = switchyard(true);+ let server = serve(app).await?;+ let dir = tempfile::tempdir()?;+ let results_dir = dir.path().join("soak-results");+ let args = args(+ &server.base_url,+ results_dir.to_str().ok_or("non-utf8 results dir")?,+ )?;++ assert_eq!(run(args).await?, 1);++ let summary: Value =+ serde_json::from_str(&std::fs::read_to_string(results_dir.join("summary.json"))?)?;+ assert_eq!(summary["passed"], json!(false));+ assert!(+ summary["error_kinds"]["invalid_response"]+ .as_u64()+ .unwrap_or(0)+ > 0+ );+ assert!(+ summary["error_kinds"]["invalid_stream"]+ .as_u64()+ .unwrap_or(0)+ > 0+ );+ assert!(+ summary["invalid_request_canary_failures"]+ .as_u64()+ .unwrap_or(0)+ > 0+ );+ Ok(())+}++#[tokio::test]+async fn nonbaseline_scenarios_all_cross_the_public_request_path() -> TestResult {+ let (app, _) = switchyard(false);+ let server = serve(app).await?;+ let dir = tempfile::tempdir()?;+ let results_dir = dir.path().join("scenario-results");+ let args = Args::try_parse_from([+ "switchyard-soak",+ "--base-url",+ &server.base_url,+ "--model",+ "soak-route",+ "--duration",+ "0.8s",+ "--concurrency",+ "4",+ "--report-interval",+ "0.2",+ "--invalid-canary-interval",+ "0",+ "--scenario",+ "long-context",+ "--scenario",+ "decode-heavy",+ "--scenario",+ "prefix-reuse",+ "--scenario",+ "mixed-traffic",+ "--scenario",+ "growing-conversation",+ "--scenario",+ "large-tool-catalog",+ "--scenario",+ "tool-call-burst",+ "--scenario",+ "stage-transitions",+ "--scenario",+ "classifier-mix",+ "--results-dir",+ results_dir.to_str().ok_or("non-utf8 results dir")?,+ ])?;++ assert_eq!(run(args).await?, 0);+ let summary: Value =+ serde_json::from_str(&std::fs::read_to_string(results_dir.join("summary.json"))?)?;+ let successes = summary["scenario_successes"]+ .as_object()+ .ok_or("scenario_successes was not an object")?;+ for scenario in [+ "long-context",+ "decode-heavy",+ "prefix-reuse",+ "mixed-traffic",+ "growing-conversation",+ "large-tool-catalog",+ "tool-call-burst",+ "stage-transitions",+ "classifier-mix",+ ] {+ assert!(successes.contains_key(scenario), "missing {scenario}");+ }+ Ok(())+}++#[tokio::test]+async fn unknown_model_is_rejected_before_the_run_starts() -> TestResult {+ let (app, _) = switchyard(false);+ let server = serve(app).await?;+ let dir = tempfile::tempdir()?;+ let results_dir = dir.path().join("soak-results");+ let args = Args::try_parse_from([+ "switchyard-soak",+ "--base-url",+ &server.base_url,+ "--model",+ "missing",+ "--duration",+ "1s",+ "--results-dir",+ results_dir.to_str().ok_or("non-utf8 results dir")?,+ ])?;++ let error = run(args).await.unwrap_err();+ assert!(error.contains("is not listed"), "{error}");+ Ok(())+}diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rsindex 87dc58b6c..23fd5140c 100644--- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs+++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs@@ -91,7 +91,13 @@ impl FormatCodec for AnthropicMessagesCodec {content,});}- if let Some(messages) = body.get("messages").and_then(Value::as_array) {+ if let Some(messages) = body.get("messages") {+ let messages = messages+ .as_array()+ .ok_or_else(|| TranslationError::InvalidType {+ path: "$.messages".to_string(),+ expected: "array",+ })?;let mut generated_id = 0;for (index, message) in messages.iter().enumerate() {let Some(message) = message.as_object() else {diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rsindex d829ca316..4b0394f43 100644--- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs+++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs@@ -72,7 +72,13 @@ impl FormatCodec for OpenAiChatCodec {..LlmRequest::default()};- if let Some(messages) = body.get("messages").and_then(Value::as_array) {+ if let Some(messages) = body.get("messages") {+ let messages = messages+ .as_array()+ .ok_or_else(|| TranslationError::InvalidType {+ path: "$.messages".to_string(),+ expected: "array",+ })?;let mut generated_id = 0;for (index, message) in messages.iter().enumerate() {let Some(message) = message.as_object() else {diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rsindex e689e5b21..c754994ce 100644--- a/crates/switchyard-translation/tests/request_translation.rs+++ b/crates/switchyard-translation/tests/request_translation.rs@@ -1782,6 +1782,18 @@ fn malformed_request_fields_are_rejected() {json!({"model": "claude", "max_tokens": "8", "messages": []}),"invalid value at $.max_tokens: expected a non-negative integer",),+ (+ "OpenAI Chat string messages",+ WireFormat::OpenAiChat,+ json!({"model": "gpt", "messages": "invalid"}),+ "expected array at $.messages",+ ),+ (+ "Anthropic string messages",+ WireFormat::AnthropicMessages,+ json!({"model": "claude", "max_tokens": 8, "messages": "invalid"}),+ "expected array at $.messages",+ ),("Responses boolean input",WireFormat::OpenAiResponses,diff --git a/docs/operations/soak_test.md b/docs/operations/soak_test.mdnew file mode 100644index 000000000..c0ce76cb4--- /dev/null+++ b/docs/operations/soak_test.md@@ -0,0 +1,342 @@+# Soak test a release candidate++A Switchyard soak test sends sustained traffic through a release-candidate+server long enough to expose failures that short tests miss. Run it for 48+hours before code freeze when a release changes libsy, the Rust server,+routing, streaming, translation, or server lifecycle behavior.++The standard test sends closed-loop traffic through:++- OpenAI Chat Completions (`/v1/chat/completions`)+- Anthropic Messages (`/v1/messages`)+- OpenAI Responses (`/v1/responses`)+- streaming and non-streaming responses+- short and long inputs, long outputs, shared prefixes, and mixed request sizes+- growing conversations, large tool catalogs, tool-call bursts, and stage transitions+- deterministic easy/hard classifier mixes++The runner also checks `/health` and `/metrics` every minute. Every five+minutes, it sends an invalid Chat Completions request, expects HTTP 400, and+then confirms the server is still live.++The scenario catalog covers these distinct pressure angles:++| Scenario | Pressure angle | What to review |+|---|---|---|+| `short-interactive` | Short prompts under fixed load, a concurrency knee, or a 10x request-rate burst | HTTP ceiling, TTFT, routing overhead, and the saturation point |+| `long-context` | 8K, 32K, and near-window inputs | TTFT, memory, and context-sensitive route failures |+| `decode-heavy` | 512-token and 1,024-token output limits | ITL, output tokens/second, and stream stability |+| `prefix-reuse` | Matched shared and unique long prefixes | Cache-sensitive TTFT and token accounting |+| `mixed-traffic` | A 70/20/10 short, medium, and long mix | p99 latency and head-of-line effects |+| `growing-conversation` | Eight cumulative conversation turns in one session | Affinity, history cost, and per-turn latency growth |+| `large-tool-catalog` | 16-tool and 64-tool JSON-schema catalogs | Serialization/routing overhead and intact tool forwarding |+| `tool-call-burst` | Eight linked assistant call/tool-result turns | Session continuity and burst handling |+| `stage-transitions` | One growing history across exploration, critical failure, and productive work | Stage-router tier changes and scorer output |+| `classifier-mix` | Deterministic 80/20 then 50/50 easy/hard requests | Target share, classifier calls/errors, and classifier latency |+| `context-overflow` | One target rejects a near-window request | Fallback to another eligible target |+| `failure-pressure` | Bounded 429, 500, malformed verdict, and truncated stream | Retry recovery, explicit terminal errors, and connection health |+| `client-cancellation` | A client timeout during a delayed response | Teardown and recovery of later traffic |++`standard` includes the core and agentic rows. Run `resilience` separately because expected+failures should not be compared as throughput samples.++## Prepare the server++Run the exact commit, build, server config, backend, and model planned for the+release. Do not use a development server in front of a different Switchyard+build.++Start the standalone server and its libsy algorithms:++```bash+cargo build --release -p switchyard-server+target/release/switchyard-server --config release-routes.toml \+ > switchyard-soak.log 2>&1 &+SOAK_SERVER_PID=$!+```++Wait for `GET http://127.0.0.1:4000/health` to return HTTP 200 with the JSON+body `{"status": "ok"}`; the runner requires both before it starts. Check+`GET http://127.0.0.1:4000/v1/models` and choose the model id that represents+the release workload. Pass that exact id with `--model`.++Run the server and test from a dedicated host, job scheduler, or terminal+multiplexer that will stay alive for the full test. Confirm that the host will+not suspend or restart and has enough disk space for the server log.++## Build the soak tester++The soak tester is a Rust binary. Build it once from the same checkout as the+release, then run it directly:++```bash+cargo build --release -p switchyard-soak+```++## Compare routing algorithm performance++Use `scripts/benchmark_routing_algorithms.py` to compare routing algorithms under the same load.+The command runs oha and AIPerf sequentially for every model id, then writes `report.md`,+`report.csv`, `report.json`, and `routing-overhead.svg` beside both tools' raw results:++```bash+python3.12 scripts/benchmark_routing_algorithms.py \+ --base-url http://127.0.0.1:4000 \+ --direct-base-url http://127.0.0.1:8100 \+ --direct-model mock/weak \+ --model noop=switchyard/noop \+ --model passthrough=switchyard/passthrough \+ --model random=switchyard/random \+ --model llm_classifier=switchyard/classifier \+ --model stage_router=switchyard/stage \+ --concurrency 100 \+ --request-count 1000 \+ --scenario-set standard \+ --load-profile fixed \+ --profile-runs 3 \+ --backend-label "release model deployment"+```++`--direct-base-url` and `--direct-model` add an AIPerf arm that calls the backend without+Switchyard. The report writes the end-to-end latency and throughput difference for every routed+arm. That difference isolates Switchyard overhead only when the direct and routed arms use the+same backend deployment, model, and response settings. Resilience scenarios have different+failure semantics, so the script does not run or compare the direct arm for them. For an+authenticated backend, pass the name of the key variable, not the key itself, with+`--direct-api-key-env NVIDIA_API_KEY`.++The Markdown report starts with one overhead row for each route and workload. Positive request or+TTFT latency means the routed request took longer than the direct request. Negative request or+token throughput means the routed path processed less work. The `standard` scenario set includes+all non-resilience request patterns exported by the Rust crate: short and long contexts, long+outputs, shared prefixes, mixed request sizes, growing conversations, large tool catalogs,+tool-call bursts, stage changes, and easy/hard classifier mixes.++When the run includes a direct backend, the Markdown report embeds `routing-overhead.svg` before+the overhead table. The two annotated heatmaps show TTFT and output-token-throughput changes for+every route and workload. Each cell shows the absolute change in milliseconds or tokens per second+and the percent change. Red cells are worse than the direct backend, and blue cells are better.+The script uses only the Python standard library to create the SVG, so the same benchmark command+reproduces the plot without a plotting package.++The Rust crate exports one AIPerf `inputs-json` file per scenario. oha reuses the first exported+`short-interactive` payload as a non-streaming fixed body and reports the raw HTTP request rate and+latency ceiling. AIPerf replays every selected streaming session and reports request latency, time+to first token (TTFT), inter-token latency (ITL), request throughput, output-token throughput, and+multi-run confidence intervals. The command saves `/v1/stats` before and after each scenario/load+cell, so the routing calls and errors total all independent repetitions in that cell. The report+also records selected-target shares, classifier calls/errors and latency, and mean routing+overhead. It runs jobs sequentially because simultaneous tools would compete for the same server+capacity.++Use the load schedules after the fixed scenario comparison establishes a baseline:++```bash+python3.12 scripts/benchmark_routing_algorithms.py \+ --base-url http://127.0.0.1:4000 \+ --model random=switchyard/random \+ --model classifier=switchyard/classifier \+ --scenario short-interactive \+ --load-profile concurrency-knee \+ --load-profile traffic-burst \+ --concurrency 128 \+ --request-rate 20+```++The concurrency knee uses bounded steps up to `--concurrency`. The traffic burst holds the base+request rate, raises it to 10 times that rate for five seconds, then returns to the base. Pass+`--load-profile all` to run fixed, knee, and burst schedules. These schedules apply to the short+baseline; they are not separate request scenarios.++To isolate routing overhead, configure every route to use the same target deployment and keep the+scenario manifest, concurrency, request count, and profile-run count fixed. Set `--tokenizer` to+the real model tokenizer when exact token counts matter. AIPerf uses deterministic sessions and a+fixed seed so each route receives the same requests.++This comparison command does not start a backend or Switchyard. Point it at a running Switchyard+server backed by real models to measure end-to-end TTFT and token throughput with real tokenization+and generated tokens. Real-model runs cost tokens and include provider queuing and model variance,+so use a dedicated deployment and repeat the run before treating a small difference as an+algorithm effect. Use the local scenario backend first for deterministic routing correctness and+overhead, then rerun the same manifest against real models for capacity claims.++## Check the local server and load tools++The local test uses a request-aware Axum backend from the soak crate. Build the server, soak tester,+and scenario backend from the commit under test:++```bash+cargo build --release -p switchyard-server -p switchyard-soak \+ --bins --example switchyard-soak-mock+```++Install [oha](https://github.com/hatoo/oha) and+[NVIDIA AIPerf](https://docs.nvidia.com/aiperf/reference/command-line-options):++```bash+cargo install oha+uv tool install --python 3.12 'aiperf==0.11.0'+```++The benchmark checks the AIPerf version before it creates an output directory. It runs each+repetition in a separate AIPerf process and calculates confidence intervals from those independent+runs. Each process starts one required record processor before profiling. The benchmark gives each+process a deadline based on its request schedule, then stops the whole process group if that deadline+expires. This prevents a startup failure from leaving a benchmark stuck at zero processed records.++Then run the local test from the repository root:++```bash+python3.12 scripts/run_local_soak_test.py \+ --duration 10s \+ --concurrency 4 \+ --request-count 100+```++The local backend waits 40 ms before the first token and 1 ms between output tokens by default.+It streams the output limit from each Rust scenario, so decode-heavy requests produce 512 or 1,024+tokens and AIPerf can measure TTFT, ITL, and token throughput separately. Use+`--mock-latency-ms` and `--mock-token-latency-ms` to change that deterministic timing model.++`--duration` controls how long the Rust soak tester runs. `--request-count` controls how many+measured requests each load tool sends to each algorithm. `--help` explains every flag. Set+`OHA_BIN`, `AIPERF_BIN`,+`SWITCHYARD_SERVER_BIN`, `SWITCHYARD_SOAK_BIN`, or `SWITCHYARD_SOAK_MOCK_BIN` when a command is not+on `PATH` or not under `target/release`.++The script checks all five commands before starting a process. A missing command prints a warning,+the build or install command, and the matching environment variable. Cargo builds the scenario+backend, but it does not install oha or AIPerf during a normal build.++The script gives each tool one job:++| Tool | Job in the local test |+|---|---|+| Scenario backend | Returns local OpenAI-compatible responses, valid easy/hard classifier verdicts, and bounded failures with no provider cost. |+| oha | Measures the non-streaming `short-interactive` HTTP baseline through every route. |+| Python route checks | Sends one ordinary Chat Completions request through each configured route before load starts. |+| AIPerf | Replays Rust-exported streaming sessions through every route and records LLM, token, response-time, and confidence results. |+| Combined report | Joins scenario, load, oha, AIPerf, and routing-counter metrics in Markdown, CSV, JSON, and an overhead plot. It keeps resilience rows separate from throughput rows. |+| `switchyard-soak` | Runs the standard scenario set while checking public API variants, server health, metrics, process use, and required results. |++`scripts/local_soak_test.toml` exercises `noop`, `random`, `passthrough`, `llm_classifier`, and+`stage_router`. It uses the accepted maximum retry count (10), a zero-weight random target, and the+upper classifier and stage thresholds (1.0). Classifier affinity is disabled so every measured+request includes the classifier call. The scenario backend returns `p_solve=1.0` for easy markers+and `p_solve=0.1` for hard markers; with the configured threshold, those requests select the weak+and strong targets respectively. The config is validated with+`switchyard-server --dry-run` before either service starts.++Run resilience cases separately so expected transport failures do not contaminate throughput+comparisons:++```bash+python3.12 scripts/benchmark_routing_algorithms.py \+ --base-url http://127.0.0.1:4000 \+ --model classifier=switchyard/classifier \+ --scenario-set resilience \+ --load-profile fixed \+ --profile-runs 1+```++`context-overflow` checks target fallback, `failure-pressure` injects bounded 429, 500, malformed+classifier, and truncated-stream cases, and `client-cancellation` uses a one-second client timeout+against a delayed response. Their expected error-rate ranges appear in the report's Resilience+section, and the command fails after writing the report when any row misses its range. The local+runner calls the scenario backend's `/reset` endpoint before each AIPerf cell so every algorithm+receives the same transient-failure sequence. When you run the comparison command directly against+that backend, pass `--scenario-backend-reset-url http://127.0.0.1:8100/reset` to preserve the same+comparison.++The runner does not add limits for target count or recent-turn history because the server does not+limit them. Rust tests cover the real limits: 4,096 saved route assignments, 100,000 response-time+samples, and 10,000 error records. Server tests cover invalid thresholds, bad classifier responses,+missing stage-router request IDs, target errors, and retry count 11, which is one above the maximum.++## Run the 48-hour test++Choose concurrency from the release capacity plan. Increase it in short+test runs until you find the highest expected steady load that remains below+the backend's rate limit. Use that load for the 48-hour run. An overload test+that spends most of its time throttled does not measure release stability.++```bash+./target/release/switchyard-soak \+ --base-url http://127.0.0.1:4000 \+ --model RELEASE_MODEL_ID \+ --duration 48h \+ --concurrency 16 \+ --server-pid "$SOAK_SERVER_PID" \+ --max-rss-growth-mib 512+```++The runner keeps 16 requests in flight until the test ends. This can generate+large usage charges against a metered backend. Use a dedicated test deployment,+estimate the request volume first with a short run, and get approval for any+paid-provider cost.++Use a five-minute run to confirm the route and result files:++```bash+./target/release/switchyard-soak \+ --base-url http://127.0.0.1:4000 \+ --model RELEASE_MODEL_ID \+ --duration 5m \+ --concurrency 4 \+ --report-interval 10+```++If the Switchyard endpoint requires a bearer token, pass the environment+variable name instead of putting the token on the command line:++```bash+export SWITCHYARD_SOAK_TOKEN="..."+./target/release/switchyard-soak \+ --api-key-env SWITCHYARD_SOAK_TOKEN \+ --model RELEASE_MODEL_ID+```++## Pass criteria++The command exits with status 0 only when:++- the requested duration completes;+- at least one inference request completes;+- the inference error rate stays at or below `--max-error-rate` (default `0`,+ which means no inference request may fail);+- every periodic liveness check passes;+- every `/metrics` read returns both Switchyard request counters;+- every requested process sample returns RSS and CPU data;+- every invalid-request recovery check passes;+- the server request counter never resets; and+- RSS growth stays within `--max-rss-growth-mib` when that limit is set.++If the release plan permits transient failures from a remote provider, set an+explicit error budget with `--max-error-rate`. Record the reason for that+exception in the release record.++The RSS limit is deployment-specific. Set it from an approved baseline for the+same model, concurrency, and worker count. Omit `--server-pid` and+`--max-rss-growth-mib` when the server runs on another host, then collect+memory and restart data from that host's monitoring system.++## Review the results++Each run creates a timestamped directory under `soak-results/`:++- `config.json` records the non-secret test inputs.+- `intervals.csv` records request rate, errors, latency percentiles, health,+ Switchyard counters, RSS, and CPU once per reporting interval. `cpu_percent`+ is the `ps` lifetime-average CPU for the process, not the interval's usage, so+ read it as a long-run average rather than a spike detector.+- `errors.jsonl` records up to 10,000 request and canary failures.+- `summary.json` records the final pass result and any failed gates.++Tail the run log to monitor progress, cumulative error rate, health, RSS, and the+`OK`, `DEGRADED`, or `STALLED` interval status while the test runs.++Before approving the release, check `intervals.csv` for late failures, falling+throughput, increasing p95 or p99 latency, and steady RSS growth. Compare the+first and last several hours, not only the run-wide averages. Attach+`summary.json`, the interval chart, the Switchyard log, the tested commit, and+the server config to the release record.diff --git a/mkdocs.yml b/mkdocs.ymlindex fcf40f715..d934b61ef 100644--- a/mkdocs.yml+++ b/mkdocs.yml@@ -32,6 +32,7 @@ nav:- Advisor-Gate Routing: routing_algorithms/advisor_gate_routing.md- Operations:- Context-Window Handling: operations/context_window.md+ - Soak Testing: operations/soak_test.md- Reference:- CLI Reference: cli_reference.md- TOML Schema: reference/toml_schema.mddiff --git a/scripts/aiperf_runner.py b/scripts/aiperf_runner.pynew file mode 100644index 000000000..f83165df1--- /dev/null+++ b/scripts/aiperf_runner.py@@ -0,0 +1,220 @@+#!/usr/bin/env python3+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0+"""Run the supported AIPerf release without allowing its startup race to hang."""++import json+import math+import os+import signal+import statistics+import subprocess+import time+from collections.abc import Sequence+from pathlib import Path++SUPPORTED_AIPERF_VERSION = "0.11.0"+PROCESS_GROUP_GRACE_SECONDS = 5+_T_CRITICAL_95 = {+ 2: 12.706,+ 3: 4.303,+ 4: 3.182,+ 5: 2.776,+ 6: 2.571,+ 7: 2.447,+ 8: 2.365,+ 9: 2.306,+ 10: 2.262,+}+_REQUIRED_STATISTICS = (+ ("error_request_count", "avg"),+ ("request_count", "avg"),+ ("request_throughput", "avg"),+)+++def validate_aiperf_version(binary: str) -> None:+ """Reject AIPerf versions whose CLI or startup behavior is not covered."""+ try:+ result = subprocess.run(+ [binary, "--version"],+ capture_output=True,+ text=True,+ check=False,+ timeout=10,+ )+ except (OSError, subprocess.TimeoutExpired) as error:+ raise RuntimeError(f"could not read the AIPerf version from {binary}: {error}") from error+ version = result.stdout.strip()+ if result.returncode != 0 or version != SUPPORTED_AIPERF_VERSION:+ found = version or result.stderr.strip() or "unknown"+ raise RuntimeError(+ f"AIPerf {found} is unsupported; install {SUPPORTED_AIPERF_VERSION} with: "+ f"uv tool install --python 3.12 'aiperf=={SUPPORTED_AIPERF_VERSION}'"+ )+++def _process_group_exists(process_group: int) -> bool:+ try:+ os.killpg(process_group, 0)+ except ProcessLookupError:+ return False+ except PermissionError:+ # A completed leader's process-group id can be reused by a process we do not own.+ return False+ return True+++def _stop_process_group(process: subprocess.Popen[bytes]) -> None:+ """Stop AIPerf and every worker process it started."""+ process_group = process.pid+ try:+ os.killpg(process_group, signal.SIGTERM)+ except ProcessLookupError:+ pass+ except PermissionError as error:+ if process.poll() is None:+ raise RuntimeError(+ f"could not stop running AIPerf process group {process_group}"+ ) from error+ return+ deadline = time.monotonic() + PROCESS_GROUP_GRACE_SECONDS+ while _process_group_exists(process_group) and time.monotonic() < deadline:+ process.poll()+ time.sleep(0.05)+ if _process_group_exists(process_group):+ try:+ os.killpg(process_group, signal.SIGKILL)+ except ProcessLookupError:+ pass+ except PermissionError as error:+ if process.poll() is None:+ raise RuntimeError(+ f"could not kill running AIPerf process group {process_group}"+ ) from error+ try:+ process.wait(timeout=PROCESS_GROUP_GRACE_SECONDS)+ except subprocess.TimeoutExpired as error:+ raise RuntimeError(f"could not reap AIPerf process {process.pid}") from error+++def run_profile(+ command: Sequence[str],+ log_path: Path,+ artifact_dir: Path,+ timeout_seconds: int,+) -> Path:+ """Run one bounded AIPerf process and return its verified export."""+ artifact_dir.parent.mkdir(parents=True, exist_ok=True)+ with log_path.open("wb") as log:+ try:+ process = subprocess.Popen(+ [*command, "--artifact-dir", str(artifact_dir)],+ stdout=log,+ stderr=subprocess.STDOUT,+ start_new_session=True,+ )+ except OSError as error:+ raise RuntimeError(f"could not start AIPerf: {error}") from error+ stopped = False+ try:+ try:+ returncode = process.wait(timeout=timeout_seconds)+ except subprocess.TimeoutExpired as error:+ timeout_message = f"AIPerf exceeded its {timeout_seconds}-second run limit"+ log.write(f"\n{timeout_message}.\n".encode())+ log.flush()+ _stop_process_group(process)+ stopped = True+ raise RuntimeError(f"{timeout_message}; see {log_path}") from error+ finally:+ if not stopped and (process.poll() is None or _process_group_exists(process.pid)):+ _stop_process_group(process)+ if returncode != 0:+ raise RuntimeError(f"AIPerf failed with status {returncode}; see {log_path}")+ export_path = artifact_dir / "profile_export_aiperf.json"+ if not export_path.is_file():+ raise RuntimeError(f"AIPerf did not write {export_path.name}; see {log_path}")+ return export_path+++def _summary(values: Sequence[float], unit: str) -> dict[str, float | str | None]:+ """Return the 95% confidence summary used by the benchmark report."""+ count = len(values)+ mean = statistics.fmean(values)+ standard_deviation = statistics.stdev(values) if count > 1 else 0.0+ standard_error = standard_deviation / math.sqrt(count)+ critical = _T_CRITICAL_95.get(count)+ margin = critical * standard_error if critical is not None else 0.0+ return {+ "mean": mean,+ "std": standard_deviation,+ "min": min(values),+ "max": max(values),+ "cv": standard_deviation / mean if mean else 0.0,+ "se": standard_error,+ "ci_low": mean - margin,+ "ci_high": mean + margin,+ "t_critical": critical,+ "unit": unit,+ }+++def aggregate_exports(exports: Sequence[Path], output_path: Path) -> Path:+ """Combine independent AIPerf exports into the aggregate schema consumed by the report."""+ if not 2 <= len(exports) <= 10:+ raise RuntimeError("AIPerf aggregation requires between 2 and 10 exports")+ collected: dict[tuple[str, str, str], list[float]] = {}+ metric_units: dict[tuple[str, str], str] = {}+ for path in exports:+ try:+ document = json.loads(path.read_text(encoding="utf-8"))+ except (OSError, json.JSONDecodeError) as error:+ raise RuntimeError(f"could not read AIPerf export {path}: {error}") from error+ if not isinstance(document, dict):+ raise RuntimeError(f"AIPerf export is not a JSON object: {path}")+ version = document.get("aiperf_version")+ if version != SUPPORTED_AIPERF_VERSION:+ raise RuntimeError(+ f"AIPerf export {path} has version {version!r}; expected {SUPPORTED_AIPERF_VERSION}"+ )+ error_count = document.get("error_request_count")+ clean_error_summary = document.get("error_summary") == []+ if error_count is None and ("error_request_count" in document or clean_error_summary):+ document["error_request_count"] = {"unit": "requests", "avg": 0.0}+ for metric_name, statistic in _REQUIRED_STATISTICS:+ metric = document.get(metric_name)+ value = metric.get(statistic) if isinstance(metric, dict) else None+ unit = metric.get("unit") if isinstance(metric, dict) else None+ if (+ not isinstance(value, int | float)+ or isinstance(value, bool)+ or not isinstance(unit, str)+ ):+ raise RuntimeError(f"AIPerf export {path} has no numeric {metric_name}.{statistic}")+ for metric_name, metric in document.items():+ if not isinstance(metric, dict) or not isinstance(metric.get("unit"), str):+ continue+ unit = metric["unit"]+ for statistic, value in metric.items():+ if statistic == "unit" or isinstance(value, bool):+ continue+ if isinstance(value, int | float):+ previous_unit = metric_units.setdefault((metric_name, statistic), unit)+ if unit != previous_unit:+ raise RuntimeError(+ "AIPerf exports disagree on the unit for "+ f"{metric_name}.{statistic}: {previous_unit!r} and {unit!r}"+ )+ collected.setdefault((metric_name, statistic, unit), []).append(float(value))+ metrics = {+ f"{metric}_{statistic}": _summary(values, unit)+ for (metric, statistic, unit), values in collected.items()+ if len(values) == len(exports)+ }+ output_path.parent.mkdir(parents=True, exist_ok=False)+ output_path.write_text(+ f"{json.dumps({'aiperf_version': SUPPORTED_AIPERF_VERSION, 'metrics': metrics}, indent=2)}\n",+ encoding="utf-8",+ )+ return output_pathdiff --git a/scripts/benchmark_routing_algorithms.py b/scripts/benchmark_routing_algorithms.pynew file mode 100755index 000000000..9a0a8baa7--- /dev/null+++ b/scripts/benchmark_routing_algorithms.py@@ -0,0 +1,1523 @@+#!/usr/bin/env python3+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0+"""Compare Switchyard routes with Rust-owned scenarios, oha, and NVIDIA AIPerf."""++import argparse+import csv+import json+import math+import os+import shutil+import subprocess+import sys+import urllib.error+import urllib.request+from collections.abc import Sequence+from dataclasses import asdict, dataclass, replace+from datetime import datetime, timezone+from pathlib import Path++if __package__:+ from .aiperf_runner import aggregate_exports, run_profile, validate_aiperf_version+ from .routing_overhead_plot import OverheadPlotRow, write_overhead_plot+else:+ from aiperf_runner import aggregate_exports, run_profile, validate_aiperf_version+ from routing_overhead_plot import OverheadPlotRow, write_overhead_plot++MAX_MATERIALIZED_REQUESTS = 1_000_000+MAX_MATERIALIZED_BYTES = 256 * 1024 * 1024+AIPERF_LIFECYCLE_ALLOWANCE_SECONDS = 120+++@dataclass(frozen=True)+class RequiredBinary:+ """One command the benchmark must find before it starts."""++ label: str+ env_var: str+ value: str+ setup: str+++@dataclass(frozen=True)+class DirectBaseline:+ """A backend endpoint that AIPerf calls without going through Switchyard."""++ base_url: str+ model: str+ api_key_env: str | None = None+++@dataclass(frozen=True)+class BenchmarkConfig:+ """Inputs shared by every algorithm comparison run."""++ base_url: str+ models: tuple[tuple[str, str], ...]+ concurrency: int+ request_count: int+ backend_label: str+ output_dir: Path+ oha_bin: str+ aiperf_bin: str+ soak_bin: str+ tokenizer: str = "builtin"+ scenario_set: str = "standard"+ scenarios: tuple[str, ...] = ()+ load_profiles: tuple[str, ...] = ("fixed",)+ profile_runs: int = 3+ prompt_bytes: int = 1024+ max_output_tokens: int = 32+ context_window_tokens: int = 32_768+ request_rate: float | None = None+ scenario_backend_reset_url: str | None = None+ direct_baseline: DirectBaseline | None = None+++@dataclass(frozen=True)+class BenchmarkArm:+ """One endpoint and model pair included in the comparison."""++ label: str+ model: str+ base_url: str+ bypasses_switchyard: bool+ api_key_env: str | None = None+++@dataclass(frozen=True)+class ScenarioDefinition:+ """One validated entry exported by switchyard-soak."""++ id: str+ group: str+ description: str+ expected: str+ expected_error_rate_min: float+ expected_error_rate_max: float+ input_file: Path+ load_profiles: tuple[dict[str, object], ...]+++@dataclass(frozen=True)+class RoutingDelta:+ """Routing counters attributable to one sequential AIPerf run."""++ model_calls: str+ model_share: str+ model_errors: str+ classifier_calls: int+ classifier_errors: int+ classifier_latency_avg_ms: float | None+ routing_overhead_avg_ms: float | None+++@dataclass(frozen=True)+class BenchmarkResult:+ """Selected load-tool and routing metrics for one comparison row."""++ algorithm: str+ model: str+ scenario: str+ scenario_description: str+ scenario_group: str+ load_profile: str+ expected_behavior: str+ expected_error_rate_min: float+ expected_error_rate_max: float+ expectation_met: bool+ oha_requests_per_second: float | None+ oha_latency_p50_ms: float | None+ oha_latency_p99_ms: float | None+ aiperf_requests_per_second: float | None+ aiperf_request_latency_p50_ms: float | None+ aiperf_ttft_p50_ms: float | None+ aiperf_ttft_p99_ms: float | None+ aiperf_itl_p50_ms: float | None+ aiperf_output_tokens_per_second: float | None+ aiperf_output_tokens_per_second_per_user_p50: float | None+ aiperf_error_requests: float+ aiperf_error_rate: float+ aiperf_request_throughput_cv: float | None+ aiperf_request_throughput_ci_low: float | None+ aiperf_request_throughput_ci_high: float | None+ selected_model_calls: str+ selected_model_share: str+ selected_model_errors: str+ classifier_calls: int+ classifier_errors: int+ classifier_latency_avg_ms: float | None+ routing_overhead_avg_ms: float | None+ bypasses_switchyard: bool+ aiperf_request_throughput_delta_pct: float | None = None+ aiperf_request_latency_p50_delta_ms: float | None = None+ aiperf_ttft_p50_delta_ms: float | None = None+ aiperf_ttft_p50_delta_pct: float | None = None+ aiperf_ttft_p99_delta_ms: float | None = None+ aiperf_output_tokens_per_second_delta: float | None = None+ aiperf_output_tokens_per_second_delta_pct: float | None = None+++def positive_int(value: str) -> int:+ """Parse a command-line integer that must be greater than zero."""+ parsed = int(value)+ if parsed <= 0:+ raise argparse.ArgumentTypeError("must be greater than zero")+ return parsed+++def positive_float(value: str) -> float:+ """Parse a finite command-line number that must be greater than zero."""+ parsed = float(value)+ if parsed <= 0 or not math.isfinite(parsed):+ raise argparse.ArgumentTypeError("must be a finite number greater than zero")+ return parsed+++def model_spec(value: str) -> tuple[str, str]:+ """Parse LABEL=MODEL without coupling the report label to a filesystem name."""+ label, separator, model = value.partition("=")+ if not separator or not label.strip() or not model.strip():+ raise argparse.ArgumentTypeError("must use LABEL=MODEL")+ return label.strip(), model.strip()+++def parser() -> argparse.ArgumentParser:+ """Build the command-line interface."""+ command = argparse.ArgumentParser(+ description=(+ "Export Rust-owned request scenarios, run oha for the short baseline and AIPerf "+ "for every selected scenario, then write one routing performance report."+ ),+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,+ )+ command.add_argument("--base-url", required=True, help="URL of a running Switchyard server")+ command.add_argument(+ "--direct-base-url",+ help="backend URL for an AIPerf baseline that bypasses Switchyard",+ )+ command.add_argument(+ "--direct-model",+ help="backend model id for the direct baseline; requires --direct-base-url",+ )+ command.add_argument(+ "--direct-api-key-env",+ help="environment variable containing the direct backend API key",+ )+ command.add_argument(+ "--model",+ action="append",+ required=True,+ type=model_spec,+ dest="models",+ metavar="LABEL=MODEL",+ help="report label and exact model id; repeat for every route to compare",+ )+ command.add_argument("--concurrency", type=positive_int, default=4)+ command.add_argument("--request-count", type=positive_int, default=100)+ command.add_argument("--tokenizer", default="builtin")+ command.add_argument(+ "--scenario-set",+ choices=("core", "agentic", "resilience", "standard", "all"),+ default="standard",+ )+ command.add_argument(+ "--scenario",+ action="append",+ default=[],+ help="exact Rust scenario id; repeat to preserve an explicit order",+ )+ command.add_argument(+ "--load-profile",+ action="append",+ choices=("fixed", "concurrency-knee", "traffic-burst", "all"),+ default=[],+ help="load schedule to run; repeat as needed",+ )+ command.add_argument(+ "--profile-runs",+ type=positive_int,+ default=3,+ help="AIPerf repetitions used for confidence intervals; maximum 10",+ )+ command.add_argument(+ "--request-rate",+ type=positive_float,+ help="base requests/second for traffic-burst; defaults to concurrency",+ )+ command.add_argument("--prompt-bytes", type=positive_int, default=1024)+ command.add_argument("--max-output-tokens", type=positive_int, default=32)+ command.add_argument("--context-window-tokens", type=positive_int, default=32_768)+ command.add_argument("--backend-label", default="unspecified backend")+ command.add_argument(+ "--scenario-backend-reset-url",+ help="optional local scenario-backend endpoint reset before each AIPerf cell",+ )+ command.add_argument("--output-dir", type=Path)+ command.add_argument("--oha-bin", default=os.environ.get("OHA_BIN", "oha"))+ command.add_argument("--aiperf-bin", default=os.environ.get("AIPERF_BIN", "aiperf"))+ command.add_argument(+ "--soak-bin",+ default=os.environ.get("SWITCHYARD_SOAK_BIN", "target/release/switchyard-soak"),+ )+ return command+++def find_binary(value: str) -> str | None:+ """Return an executable path, or None when the command cannot run."""+ if os.sep in value:+ path = Path(value).expanduser().resolve()+ if path.is_file() and os.access(path, os.X_OK):+ return str(path)+ else:+ found = shutil.which(value)+ if found is not None:+ return found+ return None+++def resolve_binaries(requirements: Sequence[RequiredBinary]) -> dict[str, str]:+ """Resolve every command and print all missing-command setup instructions."""+ resolved = {}+ missing = []+ for requirement in requirements:+ path = find_binary(requirement.value)+ if path is None:+ missing.append(requirement)+ else:+ resolved[requirement.label] = path+ for requirement in missing:+ print(+ f"warning: {requirement.label} executable not found: {requirement.value}",+ file=sys.stderr,+ )+ print(f" {requirement.setup}", file=sys.stderr)+ print(+ f" Or set {requirement.env_var}=/path/to/{Path(requirement.value).name}",+ file=sys.stderr,+ )+ if missing:+ raise RuntimeError(f"install or build the {len(missing)} missing command(s), then rerun")+ return resolved+++def run_checked(name: str, command: Sequence[str], log_path: Path) -> None:+ """Run one finite tool and keep its output in a named log file."""+ print(f"Running {name}; log: {log_path}")+ with log_path.open("w", encoding="utf-8") as log:+ result = subprocess.run(+ list(command),+ stdout=log,+ stderr=subprocess.STDOUT,+ text=True,+ check=False,+ )+ if result.returncode != 0:+ raise RuntimeError(f"{name} failed with status {result.returncode}; see {log_path}")+++def read_json_object(path: Path) -> dict[str, object]:+ """Read a JSON object and name the invalid result file in any error."""+ try:+ document = json.loads(path.read_text(encoding="utf-8"))+ except (OSError, json.JSONDecodeError) as error:+ raise RuntimeError(f"could not read JSON result {path}: {error}") from error+ if not isinstance(document, dict):+ raise RuntimeError(f"expected a JSON object in {path}")+ return document+++def capture_stats(base_url: str, path: Path) -> dict[str, object]:+ """Capture Switchyard's routing counters before or after one isolated run."""+ try:+ with urllib.request.urlopen(f"{base_url.rstrip('/')}/v1/stats", timeout=10) as response:+ document = json.loads(response.read())+ except (OSError, urllib.error.URLError, json.JSONDecodeError) as error:+ raise RuntimeError(f"could not read Switchyard stats: {error}") from error+ if not isinstance(document, dict):+ raise RuntimeError("Switchyard /v1/stats did not return a JSON object")+ path.write_text(f"{json.dumps(document, indent=2)}\n", encoding="utf-8")+ return document+++def reset_scenario_backend(url: str | None) -> None:+ """Reset local failure injection so every algorithm receives the same attempts."""+ if url is None:+ return+ request = urllib.request.Request(url, data=b"", method="POST")+ try:+ with urllib.request.urlopen(request, timeout=10) as response:+ document = json.loads(response.read())+ except (OSError, urllib.error.URLError, json.JSONDecodeError) as error:+ raise RuntimeError(f"could not reset the scenario backend: {error}") from error+ status = document.get("status") if isinstance(document, dict) else None+ if response.status != 200 or status != "reset":+ raise RuntimeError(f"scenario backend reset returned an invalid response from {url}")+++def _nested_number(document: dict[str, object], section: str, field: str) -> float:+ section_value = document.get(section)+ value = section_value.get(field) if isinstance(section_value, dict) else None+ return float(value) if isinstance(value, int | float) else 0.0+++def _histogram_delta(+ before: dict[str, object], after: dict[str, object], histogram: str+) -> float | None:+ count = _nested_number(after, histogram, "count") - _nested_number(before, histogram, "count")+ total = _nested_number(after, histogram, "total_ms") - _nested_number(+ before, histogram, "total_ms"+ )+ return total / count if count > 0 else None+++def stats_delta(before: dict[str, object], after: dict[str, object]) -> RoutingDelta:+ """Return routing and target-counter changes for one run."""+ before_value = before.get("models")+ after_value = after.get("models")+ before_models: dict[str, object] = before_value if isinstance(before_value, dict) else {}+ after_models: dict[str, object] = after_value if isinstance(after_value, dict) else {}+ classifier_before = before.get("classifier")+ classifier_after = after.get("classifier")+ before_classifier_models = (+ classifier_before.get("models") if isinstance(classifier_before, dict) else None+ )+ after_classifier_models = (+ classifier_after.get("models") if isinstance(classifier_after, dict) else None+ )+ calls = {}+ errors = {}+ for model, current in after_models.items():+ if not isinstance(model, str) or not isinstance(current, dict):+ continue+ previous = before_models.get(model)+ previous_calls = previous.get("calls", 0) if isinstance(previous, dict) else 0+ classifier_current = (+ after_classifier_models.get(model)+ if isinstance(after_classifier_models, dict)+ else None+ )+ classifier_previous = (+ before_classifier_models.get(model)+ if isinstance(before_classifier_models, dict)+ else None+ )+ classifier_call_delta = float(+ classifier_current.get("calls", 0) if isinstance(classifier_current, dict) else 0+ ) - float(+ classifier_previous.get("calls", 0) if isinstance(classifier_previous, dict) else 0+ )+ call_delta = int(+ float(current.get("calls", 0)) - float(previous_calls) - classifier_call_delta+ )+ if call_delta > 0:+ calls[model] = call_delta+ previous_errors = previous.get("errors", 0) if isinstance(previous, dict) else 0+ classifier_error_delta = float(+ classifier_current.get("errors", 0) if isinstance(classifier_current, dict) else 0+ ) - float(+ classifier_previous.get("errors", 0) if isinstance(classifier_previous, dict) else 0+ )+ error_delta = int(+ float(current.get("errors", 0)) - float(previous_errors) - classifier_error_delta+ )+ if error_delta > 0:+ errors[model] = error_delta++ classifier_count = classifier_total = 0.0+ if isinstance(after_classifier_models, dict):+ for model, current in after_classifier_models.items():+ if not isinstance(model, str) or not isinstance(current, dict):+ continue+ previous = (+ before_classifier_models.get(model)+ if isinstance(before_classifier_models, dict)+ else None+ )+ before_record = previous if isinstance(previous, dict) else {}+ classifier_count += _nested_number(current, "model_call_latency", "count")+ classifier_count -= _nested_number(before_record, "model_call_latency", "count")+ classifier_total += _nested_number(current, "model_call_latency", "total_ms")+ classifier_total -= _nested_number(before_record, "model_call_latency", "total_ms")++ total_calls = sum(calls.values())+ shares = {model: round(call_count / total_calls, 4) for model, call_count in calls.items()}+ return RoutingDelta(+ model_calls=json.dumps(calls, sort_keys=True, separators=(",", ":")),+ model_share=json.dumps(shares, sort_keys=True, separators=(",", ":")),+ model_errors=json.dumps(errors, sort_keys=True, separators=(",", ":")),+ classifier_calls=int(+ _nested_number(after, "classifier", "total_requests")+ - _nested_number(before, "classifier", "total_requests")+ ),+ classifier_errors=int(+ _nested_number(after, "classifier", "total_errors")+ - _nested_number(before, "classifier", "total_errors")+ ),+ classifier_latency_avg_ms=(+ classifier_total / classifier_count if classifier_count > 0 else None+ ),+ routing_overhead_avg_ms=_histogram_delta(before, after, "routing_overhead"),+ )+++def validate_load_profile(profile: dict[str, object], scenario_id: str) -> None:+ """Reject malformed or unbounded schedules before invoking a load generator."""+ profile_id = profile.get("id")+ kind = profile.get("kind")+ if (profile_id, kind) == ("fixed", "fixed"):+ return+ if (profile_id, kind) == ("concurrency-knee", "concurrency_knee"):+ steps = profile.get("concurrency_steps")+ if (+ not isinstance(steps, list)+ or not steps+ or not all(isinstance(step, int) and 1 <= step <= 10_000 for step in steps)+ ):+ raise RuntimeError(f"invalid concurrency knee for scenario {scenario_id}")+ return+ if (profile_id, kind) == ("traffic-burst", "traffic_burst"):+ duration = profile.get("duration_seconds")+ points = profile.get("points")+ if not isinstance(duration, int) or not 1 <= duration <= 3_600:+ raise RuntimeError(f"invalid traffic-burst duration for scenario {scenario_id}")+ if not isinstance(points, list) or not 2 <= len(points) <= 64:+ raise RuntimeError(f"invalid traffic-burst points for scenario {scenario_id}")+ previous = -1.0+ for point in points:+ time_s = point.get("time_s") if isinstance(point, dict) else None+ multiplier = point.get("rate_multiplier") if isinstance(point, dict) else None+ if (+ not isinstance(time_s, int | float)+ or not isinstance(multiplier, int | float)+ or not math.isfinite(float(time_s))+ or not math.isfinite(float(multiplier))+ or float(time_s) <= previous+ or float(time_s) > duration+ or float(multiplier) <= 0+ ):+ raise RuntimeError(f"invalid traffic-burst point for scenario {scenario_id}")+ previous = float(time_s)+ return+ raise RuntimeError(f"unknown load profile for scenario {scenario_id}: {profile_id!r}")+++def export_scenarios(config: BenchmarkConfig, index: int, model: str) -> list[ScenarioDefinition]:+ """Ask the Rust crate for the only authoritative scenario manifest."""+ output_dir = config.output_dir / "scenario-definitions" / f"algorithm-{index:02d}"+ command = [+ config.soak_bin,+ "--model",+ model,+ "--scenario-set",+ config.scenario_set,+ "--prompt-bytes",+ str(config.prompt_bytes),+ "--max-output-tokens",+ str(config.max_output_tokens),+ "--context-window-tokens",+ str(config.context_window_tokens),+ "--export-scenarios",+ str(output_dir),+ ]+ for scenario in config.scenarios:+ command.extend(("--scenario", scenario))+ run_checked(+ "scenario export",+ command,+ config.output_dir / f"scenario-export-{index:02d}.log",+ )+ document = read_json_object(output_dir / "manifest.json")+ if document.get("schema_version") != 1 or document.get("model") != model:+ raise RuntimeError(f"unsupported scenario manifest in {output_dir}")+ raw_scenarios = document.get("scenarios")+ if not isinstance(raw_scenarios, list) or not raw_scenarios:+ raise RuntimeError(f"scenario manifest has no scenarios: {output_dir}")+ definitions = []+ seen = set()+ root = output_dir.resolve()+ for entry in raw_scenarios:+ if not isinstance(entry, dict):+ raise RuntimeError(f"invalid scenario entry in {output_dir}")+ scenario_id = entry.get("id")+ group = entry.get("group")+ description = entry.get("description")+ expected = entry.get("expected")+ expected_error_rate = entry.get("expected_error_rate")+ min_error_rate = (+ expected_error_rate.get("min_rate") if isinstance(expected_error_rate, dict) else None+ )+ max_error_rate = (+ expected_error_rate.get("max_rate") if isinstance(expected_error_rate, dict) else None+ )+ input_name = entry.get("input_file")+ profiles = entry.get("load_profiles")+ if (+ not isinstance(scenario_id, str)+ or not scenario_id+ or scenario_id in seen+ or group not in {"core", "agentic", "resilience"}+ or not isinstance(description, str)+ or not description+ or not isinstance(expected, str)+ or not expected+ or not isinstance(min_error_rate, int | float)+ or not isinstance(max_error_rate, int | float)+ or not 0 <= float(min_error_rate) <= float(max_error_rate) <= 1+ or not isinstance(input_name, str)+ or not isinstance(profiles, list)+ ):+ raise RuntimeError(f"invalid scenario entry in {output_dir}")+ input_file = (output_dir / input_name).resolve()+ if not input_file.is_relative_to(root) or not input_file.is_file():+ raise RuntimeError(f"scenario input escapes or is missing from {output_dir}")+ if not all(+ isinstance(profile, dict) and isinstance(profile.get("id"), str) for profile in profiles+ ):+ raise RuntimeError(f"invalid load profile for scenario {scenario_id}")+ for profile in profiles:+ validate_load_profile(profile, scenario_id)+ seen.add(scenario_id)+ definitions.append(+ ScenarioDefinition(+ id=scenario_id,+ group=group,+ description=description,+ expected=expected,+ expected_error_rate_min=float(min_error_rate),+ expected_error_rate_max=float(max_error_rate),+ input_file=input_file,+ load_profiles=tuple(profiles),+ )+ )+ return definitions+++def selected_profiles(+ config: BenchmarkConfig, scenario: ScenarioDefinition+) -> list[dict[str, object]]:+ """Select only schedules supported by this request shape."""+ requested = set(config.load_profiles)+ if "all" in requested:+ return list(scenario.load_profiles)+ return [profile for profile in scenario.load_profiles if profile.get("id") in requested]+++def _metric(document: dict[str, object], name: str, statistic: str) -> tuple[float | None, object]:+ group = document.get(name)+ if isinstance(group, dict):+ value = group.get(statistic)+ return (float(value) if isinstance(value, int | float) else None, group.get("unit"))+ metrics = document.get("metrics")+ aggregate = metrics.get(f"{name}_{statistic}") if isinstance(metrics, dict) else None+ if isinstance(aggregate, dict):+ value = aggregate.get("mean")+ return (+ float(value) if isinstance(value, int | float) else None,+ aggregate.get("unit"),+ )+ return None, None+++def _latency(document: dict[str, object], metric: str, statistic: str, path: Path) -> float | None:+ value, unit = _metric(document, metric, statistic)+ if value is None:+ return None+ if unit == "ms":+ return value+ if unit in {"s", "seconds"}:+ return value * 1000+ raise RuntimeError(f"unsupported {metric} unit {unit!r} in {path}")+++def parse_result(+ arm: BenchmarkArm,+ scenario: ScenarioDefinition,+ load_profile: str,+ oha_path: Path | None,+ aiperf_path: Path,+ before_stats: dict[str, object],+ after_stats: dict[str, object],+) -> BenchmarkResult:+ """Combine one AIPerf run, an optional oha baseline, and routing counter deltas."""+ oha_rps = oha_p50 = oha_p99 = None+ if oha_path is not None:+ oha = read_json_object(oha_path)+ success, _unit = _metric(oha, "summary", "successRate")+ if success != 1:+ raise RuntimeError(f"oha success rate for {arm.model} was {success}; see {oha_path}")+ oha_rps, _unit = _metric(oha, "summary", "requestsPerSec")+ p50, _unit = _metric(oha, "latencyPercentiles", "p50")+ p99, _unit = _metric(oha, "latencyPercentiles", "p99")+ oha_p50 = p50 * 1000 if p50 is not None else None+ oha_p99 = p99 * 1000 if p99 is not None else None+ aiperf = read_json_object(aiperf_path)+ errors, _unit = _metric(aiperf, "error_request_count", "avg")+ error_count = errors or 0.0+ measured_requests, _unit = _metric(aiperf, "request_count", "avg")+ error_rate = (+ error_count / measured_requests+ if measured_requests is not None and measured_requests > 0+ else 1.0+ )+ expectation_met = (+ scenario.expected_error_rate_min <= error_rate <= scenario.expected_error_rate_max+ )+ throughput, _unit = _metric(aiperf, "request_throughput", "avg")+ output_throughput, _unit = _metric(aiperf, "output_token_throughput", "avg")+ user_throughput, _unit = _metric(aiperf, "e2e_output_token_throughput", "p50")+ aggregate = aiperf.get("metrics")+ throughput_stats = (+ aggregate.get("request_throughput_avg") if isinstance(aggregate, dict) else None+ )+ routing = stats_delta(before_stats, after_stats)+ return BenchmarkResult(+ algorithm=arm.label,+ model=arm.model,+ scenario=scenario.id,+ scenario_description=scenario.description,+ scenario_group=scenario.group,+ load_profile=load_profile,+ expected_behavior=scenario.expected,+ expected_error_rate_min=scenario.expected_error_rate_min,+ expected_error_rate_max=scenario.expected_error_rate_max,+ expectation_met=expectation_met,+ oha_requests_per_second=oha_rps,+ oha_latency_p50_ms=oha_p50,+ oha_latency_p99_ms=oha_p99,+ aiperf_requests_per_second=throughput,+ aiperf_request_latency_p50_ms=_latency(aiperf, "request_latency", "p50", aiperf_path),+ aiperf_ttft_p50_ms=_latency(aiperf, "time_to_first_token", "p50", aiperf_path),+ aiperf_ttft_p99_ms=_latency(aiperf, "time_to_first_token", "p99", aiperf_path),+ aiperf_itl_p50_ms=_latency(aiperf, "inter_token_latency", "p50", aiperf_path),+ aiperf_output_tokens_per_second=output_throughput,+ aiperf_output_tokens_per_second_per_user_p50=user_throughput,+ aiperf_error_requests=error_count,+ aiperf_error_rate=error_rate,+ aiperf_request_throughput_cv=(+ float(throughput_stats["cv"])+ if isinstance(throughput_stats, dict)+ and isinstance(throughput_stats.get("cv"), int | float)+ else None+ ),+ aiperf_request_throughput_ci_low=(+ float(throughput_stats["ci_low"])+ if isinstance(throughput_stats, dict)+ and isinstance(throughput_stats.get("ci_low"), int | float)+ else None+ ),+ aiperf_request_throughput_ci_high=(+ float(throughput_stats["ci_high"])+ if isinstance(throughput_stats, dict)+ and isinstance(throughput_stats.get("ci_high"), int | float)+ else None+ ),+ selected_model_calls=routing.model_calls,+ selected_model_share=routing.model_share,+ selected_model_errors=routing.model_errors,+ classifier_calls=routing.classifier_calls,+ classifier_errors=routing.classifier_errors,+ classifier_latency_avg_ms=routing.classifier_latency_avg_ms,+ routing_overhead_avg_ms=routing.routing_overhead_avg_ms,+ bypasses_switchyard=arm.bypasses_switchyard,+ )+++def _percent_change(value: float | None, baseline: float | None) -> float | None:+ if value is None or baseline is None or baseline == 0:+ return None+ return (value / baseline - 1) * 100+++def _difference(value: float | None, baseline: float | None) -> float | None:+ if value is None or baseline is None:+ return None+ return value - baseline+++def _direct_baselines(+ results: Sequence[BenchmarkResult],+) -> dict[tuple[str, str], BenchmarkResult]:+ """Index the one direct-backend result allowed for each workload."""+ baselines = {}+ for result in results:+ if not result.bypasses_switchyard:+ continue+ key = (result.scenario, result.load_profile)+ if key in baselines:+ raise RuntimeError(+ f"duplicate direct baseline for {result.scenario} {result.load_profile}"+ )+ baselines[key] = result+ return baselines+++def compare_to_direct_backend(results: Sequence[BenchmarkResult]) -> list[BenchmarkResult]:+ """Attach AIPerf deltas from the direct-backend row for each workload."""+ baselines = _direct_baselines(results)+ if not baselines:+ return list(results)++ compared = []+ for result in results:+ if result.bypasses_switchyard:+ compared.append(result)+ continue+ key = (result.scenario, result.load_profile)+ baseline = baselines.get(key)+ if baseline is None:+ if result.scenario_group != "resilience":+ raise RuntimeError(+ f"missing direct baseline for {result.scenario} {result.load_profile}"+ )+ compared.append(result)+ continue+ if result.aiperf_error_rate != 0 or baseline.aiperf_error_rate != 0:+ compared.append(result)+ continue+ compared.append(+ replace(+ result,+ aiperf_request_throughput_delta_pct=_percent_change(+ result.aiperf_requests_per_second,+ baseline.aiperf_requests_per_second,+ ),+ aiperf_request_latency_p50_delta_ms=_difference(+ result.aiperf_request_latency_p50_ms,+ baseline.aiperf_request_latency_p50_ms,+ ),+ aiperf_ttft_p50_delta_ms=_difference(+ result.aiperf_ttft_p50_ms,+ baseline.aiperf_ttft_p50_ms,+ ),+ aiperf_ttft_p50_delta_pct=_percent_change(+ result.aiperf_ttft_p50_ms,+ baseline.aiperf_ttft_p50_ms,+ ),+ aiperf_ttft_p99_delta_ms=_difference(+ result.aiperf_ttft_p99_ms,+ baseline.aiperf_ttft_p99_ms,+ ),+ aiperf_output_tokens_per_second_delta=_difference(+ result.aiperf_output_tokens_per_second,+ baseline.aiperf_output_tokens_per_second,+ ),+ aiperf_output_tokens_per_second_delta_pct=_percent_change(+ result.aiperf_output_tokens_per_second,+ baseline.aiperf_output_tokens_per_second,+ ),+ )+ )+ return compared+++def format_metric(value: float | None) -> str:+ """Render a report value without implying precision the benchmark lacks."""+ return "n/a" if value is None else f"{value:.2f}"+++def format_delta(value: float | None, suffix: str = "") -> str:+ """Render a signed direct-backend comparison."""+ return "n/a" if value is None else f"{value:+,.2f}{suffix}"+++def write_report(config: BenchmarkConfig, results: Sequence[BenchmarkResult]) -> None:+ """Write machine-readable and reviewer-readable comparison reports."""+ if not results:+ raise RuntimeError("no scenario and load-profile combinations were selected")+ rows = [asdict(result) for result in results]+ payload = {+ "generated_at": datetime.now(timezone.utc).isoformat(),+ "backend": config.backend_label,+ "base_url": config.base_url,+ "direct_baseline": (+ asdict(config.direct_baseline) if config.direct_baseline is not None else None+ ),+ "concurrency": config.concurrency,+ "request_count": config.request_count,+ "profile_runs": config.profile_runs,+ "scenario_set": config.scenario_set,+ "selected_scenarios": list(config.scenarios),+ "selected_load_profiles": list(config.load_profiles),+ "aiperf_tokenizer": config.tokenizer,+ "results": rows,+ }+ (config.output_dir / "report.json").write_text(+ f"{json.dumps(payload, indent=2)}\n", encoding="utf-8"+ )+ with (config.output_dir / "report.csv").open("w", encoding="utf-8", newline="") as output:+ writer = csv.DictWriter(output, fieldnames=list(rows[0]))+ writer.writeheader()+ writer.writerows(rows)++ lines = ["# Routing algorithm performance", ""]+ run_details = [+ f"- Backend: {config.backend_label}",+ f"- AIPerf repetitions per row: {config.profile_runs}",+ f"- Scenario set: {config.scenario_set}",+ "- Run order: algorithms ran back-to-back within each scenario and load; all jobs "+ "remained sequential",+ ]+ if config.direct_baseline is not None:+ run_details.append(+ f"- Direct baseline: `{config.direct_baseline.model}` at "+ f"`{config.direct_baseline.base_url}`; this arm bypasses Switchyard"+ )+ overhead_results = [+ result+ for result in results+ if not result.bypasses_switchyard and result.scenario_group != "resilience"+ ]+ if overhead_results:+ write_overhead_plot(+ config.output_dir / "routing-overhead.svg",+ [+ OverheadPlotRow(+ scenario=result.scenario,+ load=result.load_profile,+ route=result.algorithm,+ ttft_delta_ms=result.aiperf_ttft_p50_delta_ms,+ ttft_delta_pct=result.aiperf_ttft_p50_delta_pct,+ token_throughput_delta=(result.aiperf_output_tokens_per_second_delta),+ token_throughput_delta_pct=(+ result.aiperf_output_tokens_per_second_delta_pct+ ),+ )+ for result in overhead_results+ ],+ )+ lines.extend(+ (+ "## Routing overhead versus the direct backend",+ "",+ "The direct arm sends the same Rust-exported requests without Switchyard. These "+ "deltas isolate Switchyard overhead only when both arms use the same backend "+ "deployment, model, and response settings. Positive latency means the routed "+ "request took longer. Negative throughput means the routed path processed less "+ "work. Resilience workloads are shown later but are not compared because their "+ "failure behavior differs.",+ "",+ )+ )+ if overhead_results:+ lines.extend(+ (+ "",+ "",+ )+ )+ if not overhead_results:+ lines.append("No successful direct-backend comparisons were available.")+ else:+ lines.extend(+ (+ "| Workload | Route | Load | Request p50 change (ms) | "+ "TTFT p50 change (ms) | TTFT change (%) | Request throughput change (%) | "+ "Token throughput change (tokens/s) | Token throughput change (%) |",+ "|---|---|---|---:|---:|---:|---:|---:|---:|",+ )+ )+ for result in overhead_results:+ lines.append(+ f"| {result.scenario.replace('-', ' ')} | {result.algorithm} | "+ f"{result.load_profile} | "+ f"{format_delta(result.aiperf_request_latency_p50_delta_ms, ' ms')} | "+ f"{format_delta(result.aiperf_ttft_p50_delta_ms, ' ms')} | "+ f"{format_delta(result.aiperf_ttft_p50_delta_pct, '%')} | "+ f"{format_delta(result.aiperf_request_throughput_delta_pct, '%')} | "+ f"{format_delta(result.aiperf_output_tokens_per_second_delta, ' tokens/s')} | "+ f"{format_delta(result.aiperf_output_tokens_per_second_delta_pct, '%')} |"+ )+ lines.extend(+ (+ "",+ "Use the confidence intervals in the detailed results before treating a small "+ "throughput change as meaningful. A small negative latency or positive "+ "throughput delta is run-to-run noise when those intervals overlap. Request and "+ "token throughput percentage changes match when every arm emits the same fixed "+ "output length; the detailed table shows the absolute token rates.",+ "",+ "## Workloads measured",+ "",+ "| Workload | Request pattern |",+ "|---|---|",+ )+ )+ seen_scenarios = set()+ for result in overhead_results:+ if result.scenario in seen_scenarios:+ continue+ seen_scenarios.add(result.scenario)+ lines.append(f"| {result.scenario.replace('-', ' ')} | {result.scenario_description} |")+ lines.extend(("", "## Run details", "", *run_details))+ lines.extend(+ (+ "",+ "## Throughput and latency",+ "",+ "| Algorithm | Scenario | Load | oha req/s | AIPerf req/s | request p50 ms | "+ "TTFT p50 ms | TTFT p99 ms | ITL p50 ms | output tok/s | error rate | gate |",+ "|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---|",+ )+ )+ for result in results:+ if result.scenario_group == "resilience":+ continue+ lines.append(+ f"| {result.algorithm} | {result.scenario} | {result.load_profile} | "+ f"{format_metric(result.oha_requests_per_second)} | "+ f"{format_metric(result.aiperf_requests_per_second)} | "+ f"{format_metric(result.aiperf_request_latency_p50_ms)} | "+ f"{format_metric(result.aiperf_ttft_p50_ms)} | "+ f"{format_metric(result.aiperf_ttft_p99_ms)} | "+ f"{format_metric(result.aiperf_itl_p50_ms)} | "+ f"{format_metric(result.aiperf_output_tokens_per_second)} | "+ f"{result.aiperf_error_rate:.2%} | "+ f"{'PASS' if result.expectation_met else 'FAIL'} |"+ )+ if config.profile_runs > 1:+ lines.extend(+ (+ "",+ "## Repeatability",+ "",+ "The script calculates variation and confidence intervals across independent "+ "AIPerf runs. "+ "Treat a small throughput delta as noise when these intervals overlap.",+ "",+ "| Algorithm | Scenario | Load | req/s average | req/s CV | req/s 95% CI |",+ "|---|---|---|---:|---:|---:|",+ )+ )+ for result in results:+ if result.scenario_group == "resilience":+ continue+ confidence_interval = (+ "n/a"+ if result.aiperf_request_throughput_ci_low is None+ or result.aiperf_request_throughput_ci_high is None+ else f"{result.aiperf_request_throughput_ci_low:.2f}–"+ f"{result.aiperf_request_throughput_ci_high:.2f}"+ )+ coefficient_of_variation = (+ "n/a"+ if result.aiperf_request_throughput_cv is None+ else f"{result.aiperf_request_throughput_cv:.2%}"+ )+ lines.append(+ f"| {result.algorithm} | {result.scenario} | {result.load_profile} | "+ f"{format_metric(result.aiperf_requests_per_second)} | "+ f"{coefficient_of_variation} | {confidence_interval} |"+ )+ lines.extend(+ (+ "",+ "## Routing behavior",+ "",+ "| Algorithm | Scenario | Load | selected target calls | selected target share | target errors | "+ "classifier calls | classifier errors | classifier avg ms | "+ "Switchyard routing time avg ms |",+ "|---|---|---|---|---|---|---:|---:|---:|---:|",+ )+ )+ for result in results:+ if result.bypasses_switchyard:+ continue+ lines.append(+ f"| {result.algorithm} | {result.scenario} | {result.load_profile} | "+ f"`{result.selected_model_calls}` | `{result.selected_model_share}` | "+ f"`{result.selected_model_errors}` | "+ f"{result.classifier_calls} | {result.classifier_errors} | "+ f"{format_metric(result.classifier_latency_avg_ms)} | "+ f"{format_metric(result.routing_overhead_avg_ms)} |"+ )++ resilience = [result for result in results if result.scenario_group == "resilience"]+ if resilience:+ lines.extend(+ (+ "",+ "## Resilience",+ "",+ "| Algorithm | Scenario | Load | client error rate | expected range | gate | "+ "target errors | Expected behavior |",+ "|---|---|---|---:|---|---|---|---|",+ )+ )+ for result in resilience:+ if result.bypasses_switchyard:+ continue+ lines.append(+ f"| {result.algorithm} | {result.scenario} | {result.load_profile} | "+ f"{result.aiperf_error_rate:.2%} | "+ f"{result.expected_error_rate_min:.2%}–{result.expected_error_rate_max:.2%} | "+ f"{'PASS' if result.expectation_met else 'FAIL'} | "+ f"`{result.selected_model_errors}` | "+ f"{result.expected_behavior} |"+ )+ lines.extend(+ (+ "",+ "oha runs only for the fixed short-interactive baseline. AIPerf replays the same "+ "Rust-exported request payload for streaming, session-aware LLM metrics. Compare "+ "algorithms within the same scenario and load row.",+ "",+ "The local scenario backend isolates Switchyard routing and protocol overhead. It "+ "does not predict model capacity. Repeat the same command against routes backed by "+ "one real deployment to include tokenization, model execution, and provider queuing.",+ "",+ )+ )+ (config.output_dir / "report.md").write_text("\n".join(lines), encoding="utf-8")+++def run_oha(+ config: BenchmarkConfig,+ label: str,+ scenario: ScenarioDefinition,+ output_dir: Path,+) -> Path:+ """Run oha only for the fixed short-interactive baseline."""+ inputs = read_json_object(scenario.input_file)+ data = inputs.get("data")+ try:+ payload = data[0]["payloads"][0] # type: ignore[index]+ except (IndexError, KeyError, TypeError) as error:+ raise RuntimeError(f"invalid AIPerf input file {scenario.input_file}") from error+ if not isinstance(payload, dict):+ raise RuntimeError(f"invalid AIPerf payload in {scenario.input_file}")+ body = dict(payload)+ body["stream"] = False+ request_path = output_dir / f"{label}-request.json"+ result_path = output_dir / f"{label}.json"+ request_path.write_text(json.dumps(body), encoding="utf-8")+ run_checked(+ f"oha {label}",+ [+ config.oha_bin,+ "-n",+ str(config.request_count),+ "-c",+ str(config.concurrency),+ "--latency-correction",+ "--no-tui",+ "--method",+ "POST",+ "-T",+ "application/json",+ "-D",+ str(request_path),+ "--output-format",+ "json",+ "--output",+ str(result_path),+ f"{config.base_url.rstrip('/')}/v1/chat/completions",+ ],+ output_dir / f"{label}.log",+ )+ return result_path+++def materialize_aiperf_input(+ scenario: ScenarioDefinition,+ output_path: Path,+ minimum_request_count: int,+ namespace: str = "load",+) -> Path:+ """Repeat Rust-owned sessions with unique ids so AIPerf never wraps the dataset."""+ if minimum_request_count > MAX_MATERIALIZED_REQUESTS:+ raise RuntimeError(+ f"AIPerf input needs {minimum_request_count} requests; maximum is "+ f"{MAX_MATERIALIZED_REQUESTS}"+ )+ document = read_json_object(scenario.input_file)+ data = document.get("data")+ if not isinstance(data, list) or not data:+ raise RuntimeError(f"AIPerf input has no sessions: {scenario.input_file}")++ validated = []+ requests_per_replica = 0+ bytes_per_replica = 0+ for raw_session in data:+ if not isinstance(raw_session, dict):+ raise RuntimeError(f"invalid AIPerf session in {scenario.input_file}")+ session_id = raw_session.get("session_id")+ payloads = raw_session.get("payloads")+ if not isinstance(session_id, str) or not isinstance(payloads, list) or not payloads:+ raise RuntimeError(f"invalid AIPerf session in {scenario.input_file}")+ validated.append((raw_session, session_id, len(payloads)))+ requests_per_replica += len(payloads)+ bytes_per_replica += len(json.dumps(raw_session).encode()) + 64+ replica_count = math.ceil(minimum_request_count / requests_per_replica)+ estimated_bytes = replica_count * bytes_per_replica+ if estimated_bytes > MAX_MATERIALIZED_BYTES:+ raise RuntimeError(+ f"AIPerf input would use about {estimated_bytes} bytes; maximum is "+ f"{MAX_MATERIALIZED_BYTES}"+ )++ sessions = []+ request_count = 0+ replica = 0+ for _replica_index in range(replica_count):+ for raw_session, session_id, payload_count in validated:+ sessions.append(+ {+ **raw_session,+ "session_id": f"{session_id}-{namespace}-{replica:06d}",+ }+ )+ request_count += payload_count+ replica += 1+ if request_count >= minimum_request_count:+ break+ if request_count >= minimum_request_count:+ break+ output_path.write_text(json.dumps({"data": sessions}), encoding="utf-8")+ return output_path+++def profile_request_count(config: BenchmarkConfig, profile: dict[str, object]) -> int:+ """Return a conservative request bound for one AIPerf schedule."""+ if profile.get("kind") != "traffic_burst":+ return config.request_count+ duration = profile.get("duration_seconds")+ points = profile.get("points")+ if not isinstance(duration, int) or not isinstance(points, list):+ raise RuntimeError("traffic-burst profile is missing its duration or points")+ parsed_points = []+ for point in points:+ if not isinstance(point, dict):+ raise RuntimeError("traffic-burst profile has an invalid point")+ time_s = point.get("time_s")+ multiplier = point.get("rate_multiplier")+ if not isinstance(time_s, int | float) or not isinstance(multiplier, int | float):+ raise RuntimeError("traffic-burst profile has an invalid point")+ parsed_points.append((float(time_s), float(multiplier)))+ request_multiplier_seconds = sum(+ (right_time - left_time) * (left_rate + right_rate) / 2+ for (left_time, left_rate), (right_time, right_rate) in zip(+ parsed_points, parsed_points[1:], strict=False+ )+ )+ return math.ceil(request_multiplier_seconds * (config.request_rate or config.concurrency))+++def aiperf_timeout_seconds(+ config: BenchmarkConfig,+ scenario: ScenarioDefinition,+ profile: dict[str, object],+ concurrency: int | None,+) -> int:+ """Bound one process by its schedule and per-request timeout."""+ request_timeout = 1 if scenario.id == "client-cancellation" else 30+ if profile.get("kind") == "traffic_burst":+ duration = profile.get("duration_seconds")+ if not isinstance(duration, int):+ raise RuntimeError(f"traffic-burst profile has no duration for {scenario.id}")+ workload_seconds = duration + request_timeout+ else:+ effective_concurrency = concurrency or config.concurrency+ workload_seconds = math.ceil(config.request_count / effective_concurrency) * request_timeout+ return AIPERF_LIFECYCLE_ALLOWANCE_SECONDS + workload_seconds+++def run_aiperf(+ config: BenchmarkConfig,+ arm: BenchmarkArm,+ scenario: ScenarioDefinition,+ profile: dict[str, object],+ output_dir: Path,+ artifact_label: str,+ concurrency: int | None = None,+) -> Path:+ """Run independent AIPerf repetitions using Rust-exported inputs-json datasets."""+ artifact_root = output_dir / artifact_label+ effective_concurrency = concurrency or config.concurrency+ load_arguments: list[str] = []+ kind = profile.get("kind")+ if kind == "traffic_burst":+ points = profile.get("points")+ if not isinstance(points, list):+ raise RuntimeError(f"traffic-burst profile has no points for {scenario.id}")+ base_rate = config.request_rate or float(config.concurrency)+ rate_points = []+ for point in points:+ if not isinstance(point, dict):+ raise RuntimeError(f"invalid traffic-burst point for {scenario.id}")+ rate_points.append(+ {+ "time_s": point["time_s"],+ "qps": base_rate * float(point["rate_multiplier"]),+ }+ )+ series_path = output_dir / f"{artifact_label}-request-rate.json"+ series_path.write_text(+ f"{json.dumps({'points': rate_points}, indent=2)}\n", encoding="utf-8"+ )+ load_arguments.extend(+ ("--request-rate-series", str(series_path), "--arrival-pattern", "constant")+ )+ load_arguments.extend(("--benchmark-duration", str(profile.get("duration_seconds"))))+ load_arguments.extend(("--concurrency", str(config.concurrency)))+ else:+ load_arguments.extend(("--concurrency", str(effective_concurrency)))+ load_arguments.extend(("--request-count", str(config.request_count)))++ exports: list[Path] = []+ for trial in range(1, config.profile_runs + 1):+ run_label = f"{artifact_label}-run-{trial:02d}"+ input_path = materialize_aiperf_input(+ scenario,+ output_dir / f"{run_label}-inputs.json",+ profile_request_count(config, profile),+ namespace=f"{artifact_label}-run-{trial:02d}",+ )+ command = [+ config.aiperf_bin,+ "profile",+ "--model",+ arm.model,+ "--url",+ arm.base_url,+ "--endpoint-type",+ "chat",+ "--streaming",+ "--tokenizer",+ config.tokenizer,+ "--custom-dataset-type",+ "inputs-json",+ "--input-file",+ str(input_path),+ "--session-header",+ "x-switchyard-session-id",+ "--random-seed",+ "42",+ "--num-profile-runs",+ "1",+ "--record-processors",+ "1",+ "--request-timeout-seconds",+ "1" if scenario.id == "client-cancellation" else "30",+ "--ui",+ "none",+ *load_arguments,+ ]+ if arm.api_key_env is not None:+ command.extend(("--api-key", f"${{{arm.api_key_env}}}"))+ trial_root = (+ artifact_root / "profile_runs" / f"run_{trial:04d}"+ if config.profile_runs > 1+ else artifact_root+ )+ exports.append(+ run_profile(+ command,+ output_dir / f"{run_label}.log",+ trial_root,+ aiperf_timeout_seconds(config, scenario, profile, concurrency),+ )+ )+ if len(exports) == 1:+ return exports[0]+ aggregate_path: Path = aggregate_exports(+ exports,+ artifact_root / "aggregate" / "profile_export_aiperf_aggregate.json",+ )+ return aggregate_path+++def run_benchmark(config: BenchmarkConfig) -> Path:+ """Run selected scenarios for every algorithm and return the report directory."""+ if not config.models:+ raise RuntimeError("at least one algorithm model is required")+ if config.profile_runs > 10:+ raise RuntimeError("--profile-runs must be between 1 and 10")+ validate_aiperf_version(config.aiperf_bin)+ arms = [BenchmarkArm(label, model, config.base_url, False) for label, model in config.models]+ if config.direct_baseline is not None:+ arms.insert(+ 0,+ BenchmarkArm(+ "direct-backend",+ config.direct_baseline.model,+ config.direct_baseline.base_url,+ True,+ config.direct_baseline.api_key_env,+ ),+ )+ labels = [arm.label for arm in arms]+ if len(labels) != len(set(labels)):+ raise RuntimeError("algorithm labels must be unique")+ config.output_dir.mkdir(parents=True, exist_ok=False)+ oha_dir = config.output_dir / "oha"+ aiperf_dir = config.output_dir / "aiperf"+ oha_dir.mkdir()+ aiperf_dir.mkdir()+ results = []+ definition_sets = [export_scenarios(config, index, arm.model) for index, arm in enumerate(arms)]+ expected_ids = [scenario.id for scenario in definition_sets[0]]+ for definitions in definition_sets[1:]:+ if [scenario.id for scenario in definitions] != expected_ids:+ raise RuntimeError("scenario exports differ between algorithms")++ for scenario_index, baseline_scenario in enumerate(definition_sets[0]):+ for profile in selected_profiles(config, baseline_scenario):+ profile_id = str(profile["id"])+ concurrencies: list[int | None] = [None]+ if profile.get("kind") == "concurrency_knee":+ steps = profile.get("concurrency_steps")+ if not isinstance(steps, list) or not all(isinstance(step, int) for step in steps):+ raise RuntimeError(f"invalid concurrency knee for {baseline_scenario.id}")+ concurrencies = sorted(+ set(+ [step for step in steps if step <= config.concurrency]+ + [config.concurrency]+ )+ )+ for concurrency in concurrencies:+ load_label = profile_id if concurrency is None else f"{profile_id}@{concurrency}"+ for index, arm in enumerate(arms):+ scenario = definition_sets[index][scenario_index]+ if arm.bypasses_switchyard and scenario.group == "resilience":+ continue+ artifact_label = (+ f"algorithm-{index:02d}-{scenario.id}-{load_label.replace('@', '-')}"+ )+ oha_path = None+ if (+ not arm.bypasses_switchyard+ and scenario.id == "short-interactive"+ and profile_id == "fixed"+ ):+ oha_path = run_oha(+ config,+ artifact_label,+ scenario,+ oha_dir,+ )+ reset_scenario_backend(config.scenario_backend_reset_url)+ before = (+ {}+ if arm.bypasses_switchyard+ else capture_stats(+ config.base_url,+ aiperf_dir / f"{artifact_label}-stats-before.json",+ )+ )+ aiperf_path = run_aiperf(+ config,+ arm,+ scenario,+ profile,+ aiperf_dir,+ artifact_label,+ concurrency,+ )+ after = (+ {}+ if arm.bypasses_switchyard+ else capture_stats(+ config.base_url,+ aiperf_dir / f"{artifact_label}-stats-after.json",+ )+ )+ results.append(+ parse_result(+ arm,+ scenario,+ load_label,+ oha_path,+ aiperf_path,+ before,+ after,+ )+ )+ results = compare_to_direct_backend(results)+ write_report(config, results)+ failures = [result for result in results if not result.expectation_met]+ if failures:+ raise RuntimeError(+ f"{len(failures)} benchmark row(s) missed their error-rate gate; "+ f"see {config.output_dir / 'report.md'}"+ )+ return config.output_dir+++def default_output_dir() -> Path:+ """Choose a new timestamped directory under the repository."""+ repo_root = Path(__file__).resolve().parents[1]+ stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")+ return repo_root / "routing-benchmark-results" / stamp+++def main(argv: Sequence[str] | None = None) -> int:+ """Parse arguments, run the comparison, and print one actionable failure."""+ command = parser()+ args = command.parse_args(argv)+ if (args.direct_base_url is None) != (args.direct_model is None):+ command.error("--direct-base-url and --direct-model must be used together")+ if args.direct_api_key_env is not None and args.direct_base_url is None:+ command.error("--direct-api-key-env requires --direct-base-url and --direct-model")+ try:+ binaries = resolve_binaries(+ (+ RequiredBinary(+ "oha", "OHA_BIN", args.oha_bin, "Install oha with: cargo install oha"+ ),+ RequiredBinary(+ "AIPerf",+ "AIPERF_BIN",+ args.aiperf_bin,+ "Install AIPerf with: uv tool install --python 3.12 'aiperf==0.11.0'",+ ),+ RequiredBinary(+ "switchyard-soak",+ "SWITCHYARD_SOAK_BIN",+ args.soak_bin,+ "Build the scenario exporter with: cargo build --release -p switchyard-soak",+ ),+ )+ )+ output_dir = run_benchmark(+ BenchmarkConfig(+ base_url=args.base_url,+ models=tuple(args.models),+ concurrency=args.concurrency,+ request_count=args.request_count,+ tokenizer=args.tokenizer,+ scenario_set=args.scenario_set,+ scenarios=tuple(args.scenario),+ load_profiles=tuple(args.load_profile or ["fixed"]),+ profile_runs=args.profile_runs,+ prompt_bytes=args.prompt_bytes,+ max_output_tokens=args.max_output_tokens,+ context_window_tokens=args.context_window_tokens,+ request_rate=args.request_rate,+ backend_label=args.backend_label,+ output_dir=(args.output_dir or default_output_dir()).resolve(),+ oha_bin=binaries["oha"],+ aiperf_bin=binaries["AIPerf"],+ soak_bin=binaries["switchyard-soak"],+ scenario_backend_reset_url=args.scenario_backend_reset_url,+ direct_baseline=(+ DirectBaseline(+ args.direct_base_url,+ args.direct_model,+ args.direct_api_key_env,+ )+ if args.direct_base_url is not None and args.direct_model is not None+ else None+ ),+ )+ )+ except (OSError, RuntimeError) as error:+ print(f"routing algorithm benchmark failed: {error}", file=sys.stderr)+ return 1+ print(f"Routing algorithm benchmark passed; report: {output_dir / 'report.md'}")+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/scripts/local_soak_test.toml b/scripts/local_soak_test.tomlnew file mode 100644index 000000000..967095094--- /dev/null+++ b/scripts/local_soak_test.toml@@ -0,0 +1,57 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++# Local routes used by run_local_soak_test.py; no provider credentials are required.++schema_version = 1++[llm_clients.mock]+format = "openai_chat"+base_url = "http://127.0.0.1:8100/v1"+max_retries = 10++[targets.classifier]+id = "mock/classifier"+llm_client = "mock"++[targets.strong]+id = "mock/strong"+llm_client = "mock"++[targets.weak]+id = "mock/weak"+llm_client = "mock"++[routes.noop]+id = "switchyard/noop"+type = "noop"++[routes.random]+id = "switchyard/random"+type = "random"+targets = ["weak", "strong"]+weights = [1, 0]+seed = 42++[routes.passthrough]+id = "switchyard/passthrough"+type = "passthrough"+target = "weak"++[routes.classifier]+id = "switchyard/classifier"+type = "llm_classifier"+mode = "capability"+classifier_target = "classifier"+strong_target = "strong"+weak_target = "weak"+base_threshold = 1.0+max_output_tokens = 1++[routes.stage]+id = "switchyard/stage"+type = "stage_router"+capable_target = "strong"+efficient_target = "weak"+picker = "efficient_first"+confidence_threshold = 1.0diff --git a/scripts/routing_overhead_plot.py b/scripts/routing_overhead_plot.pynew file mode 100644index 000000000..65c6fd2a8--- /dev/null+++ b/scripts/routing_overhead_plot.py@@ -0,0 +1,291 @@+#!/usr/bin/env python3+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0+"""Render routing-overhead results as a dependency-free SVG heatmap."""++import math+from collections.abc import Sequence+from dataclasses import dataclass+from html import escape+from pathlib import Path+from typing import TypeGuard+++@dataclass(frozen=True)+class OverheadPlotRow:+ """One route and workload comparison against the direct backend."""++ scenario: str+ load: str+ route: str+ ttft_delta_ms: float | None+ ttft_delta_pct: float | None+ token_throughput_delta: float | None+ token_throughput_delta_pct: float | None+++@dataclass(frozen=True)+class _Panel:+ metric: str+ title: str+ subtitle: str+ primary_suffix: str+ primary_values: tuple[float | None, ...]+ percent_values: tuple[float | None, ...]+ higher_is_better: bool+++_BACKGROUND = (255, 255, 255)+_BETTER = (42, 111, 174)+_WORSE = (197, 52, 52)+_NEUTRAL = "#f3f5f7"+++def _finite(value: float | None) -> TypeGuard[float]:+ return value is not None and math.isfinite(value)+++def _fill(value: float | None, maximum: float, higher_is_better: bool) -> str:+ if not _finite(value) or value == 0:+ return _NEUTRAL+ worse = value < 0 if higher_is_better else value > 0+ target = _WORSE if worse else _BETTER+ intensity = 0.16 + 0.64 * min(abs(value) / maximum, 1.0)+ channels = (+ round(background + (foreground - background) * intensity)+ for background, foreground in zip(_BACKGROUND, target, strict=True)+ )+ red, green, blue = channels+ return f"#{red:02x}{green:02x}{blue:02x}"+++def _format_value(value: float | None, suffix: str) -> str:+ if not _finite(value):+ return "n/a"+ return f"{value:+,.2f}{suffix}"+++def _text(+ x: float,+ y: float,+ value: str,+ anchor: str = "start",+ size: int = 13,+ weight: int = 400,+ fill: str = "#1d2733",+) -> str:+ return (+ f'<text x="{x}" y="{y}" text-anchor="{anchor}" font-family="system-ui, '+ f'-apple-system, sans-serif" font-size="{size}" font-weight="{weight}" '+ f'fill="{fill}">{escape(value)}</text>'+ )+++def write_overhead_plot(path: Path, rows: Sequence[OverheadPlotRow]) -> None:+ """Write an annotated SVG overview for every supplied route and workload."""+ if not rows:+ raise ValueError("an overhead plot requires at least one row")++ workloads = list(dict.fromkeys((row.scenario, row.load) for row in rows))+ routes = list(dict.fromkeys(row.route for row in rows))+ by_key: dict[tuple[str, str, str], OverheadPlotRow] = {}+ for row in rows:+ key = (row.scenario, row.load, row.route)+ if key in by_key:+ raise RuntimeError(+ f"duplicate overhead plot row for {row.scenario}, {row.load}, and {row.route}"+ )+ by_key[key] = row++ ordered_rows = [+ by_key.get((scenario, load, route)) for scenario, load in workloads for route in routes+ ]+ panels = (+ _Panel(+ "ttft_p50_delta_ms",+ "TTFT p50 overhead",+ "Each cell shows the absolute change in ms and the percent change; lower is better",+ " ms",+ tuple(row.ttft_delta_ms if row is not None else None for row in ordered_rows),+ tuple(row.ttft_delta_pct if row is not None else None for row in ordered_rows),+ False,+ ),+ _Panel(+ "output_tokens_per_second_delta",+ "Output token throughput overhead",+ "Each cell shows the absolute change in tokens/s and the percent change; higher is better",+ " tokens/s",+ tuple(row.token_throughput_delta if row is not None else None for row in ordered_rows),+ tuple(+ row.token_throughput_delta_pct if row is not None else None for row in ordered_rows+ ),+ True,+ ),+ )++ label_width = 250+ cell_width = 170+ row_height = 50+ panel_header_height = 76+ panel_gap = 52+ top = 112+ right = 24+ panel_height = panel_header_height + len(workloads) * row_height+ width = label_width + len(routes) * cell_width + right+ height = top + len(panels) * panel_height + panel_gap + 28+ center = width / 2+ legend_x = center - 260++ svg = [+ (+ f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" '+ f'viewBox="0 0 {width} {height}" role="img" '+ 'aria-labelledby="routing-overhead-title routing-overhead-description">'+ ),+ '<title id="routing-overhead-title">Routing overhead versus the direct backend</title>',+ (+ '<desc id="routing-overhead-description">Two annotated heatmaps compare median '+ "time to first token in milliseconds and output token throughput in tokens per "+ "second for each route and workload. Every cell also gives the percent change. Red "+ "cells are worse than the direct backend and blue cells are better. Darker cells "+ "show a larger percent change within each heatmap.</desc>"+ ),+ f'<rect width="{width}" height="{height}" fill="#ffffff"/>',+ _text(+ center,+ 31,+ "Routing overhead versus the direct backend",+ anchor="middle",+ size=22,+ weight=700,+ ),+ _text(+ center,+ 55,+ "Measures Switchyard overhead when both arms use the same deployment, model, and settings",+ anchor="middle",+ size=13,+ fill="#596777",+ ),+ f'<rect x="{legend_x}" y="72" width="18" height="14" rx="2" fill="#e6a9a9"/>',+ _text(legend_x + 24, 84, "worse", size=12, fill="#596777"),+ f'<rect x="{legend_x + 80}" y="72" width="18" height="14" rx="2" fill="#a8c7e2"/>',+ _text(legend_x + 104, 84, "better", size=12, fill="#596777"),+ f'<rect x="{legend_x + 163}" y="72" width="18" height="14" rx="2" fill="#f3f5f7"/>',+ _text(+ legend_x + 187,+ 84,+ "no change / unavailable; darker = larger % change",+ size=12,+ fill="#596777",+ ),+ ]++ for panel_index, panel in enumerate(panels):+ panel_y = top + panel_index * (panel_height + panel_gap)+ svg.append(_text(center, panel_y + 18, panel.title, anchor="middle", size=16, weight=650))+ svg.append(+ _text(+ center,+ panel_y + 39,+ panel.subtitle,+ anchor="middle",+ size=12,+ fill="#596777",+ )+ )+ svg.append(+ _text(+ label_width / 2,+ panel_y + 66,+ "Workload / load",+ anchor="middle",+ size=12,+ weight=600,+ )+ )+ for route_index, route in enumerate(routes):+ cell_x = label_width + route_index * cell_width+ svg.append(+ _text(+ cell_x + cell_width / 2,+ panel_y + 66,+ route.replace("-", " ").replace("_", " "),+ anchor="middle",+ size=12,+ weight=600,+ )+ )++ finite_values = [abs(value) for value in panel.percent_values if _finite(value)]+ maximum = max(finite_values, default=1.0) or 1.0+ row_y = panel_y + panel_header_height+ for workload_index, (scenario, load) in enumerate(workloads):+ y = row_y + workload_index * row_height+ svg.append(+ _text(+ label_width / 2,+ y + 21,+ scenario.replace("-", " "),+ anchor="middle",+ size=12,+ fill="#344150",+ )+ )+ svg.append(+ _text(+ label_width / 2,+ y + 38,+ load.replace("-", " "),+ anchor="middle",+ size=11,+ fill="#697786",+ )+ )+ for route_index in range(len(routes)):+ value_index = workload_index * len(routes) + route_index+ primary_value = panel.primary_values[value_index]+ percent_value = panel.percent_values[value_index]+ x = label_width + route_index * cell_width+ cell_title = (+ f"{scenario}, {load}, {routes[route_index]}: "+ f"{_format_value(primary_value, panel.primary_suffix)} "+ f"({_format_value(percent_value, '%')})"+ )+ svg.append(+ f'<g data-scenario="{escape(scenario, quote=True)}" '+ f'data-load="{escape(load, quote=True)}" '+ f'data-route="{escape(routes[route_index], quote=True)}" '+ f'data-metric="{panel.metric}">'+ )+ svg.append(f"<title>{escape(cell_title)}</title>")+ svg.append(+ f'<rect x="{x + 2}" y="{y + 2}" width="{cell_width - 4}" '+ f'height="{row_height - 4}" rx="3" '+ f'fill="{_fill(percent_value, maximum, panel.higher_is_better)}" '+ 'stroke="#d9e0e7"/>'+ )+ svg.append(+ _text(+ x + cell_width / 2,+ y + 21,+ _format_value(primary_value, panel.primary_suffix),+ anchor="middle",+ size=12,+ weight=600,+ )+ )+ svg.append(+ _text(+ x + cell_width / 2,+ y + 38,+ f"({_format_value(percent_value, '%')})",+ anchor="middle",+ size=11,+ fill="#44515f",+ )+ )+ svg.append("</g>")++ svg.append("</svg>")+ path.write_text("\n".join(svg) + "\n", encoding="utf-8")diff --git a/scripts/run_local_soak_test.py b/scripts/run_local_soak_test.pynew file mode 100755index 000000000..32decdae3--- /dev/null+++ b/scripts/run_local_soak_test.py@@ -0,0 +1,369 @@+#!/usr/bin/env python3+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0+"""Run every Switchyard route with the local scenario backend and load tools."""++import argparse+import json+import os+import shutil+import subprocess+import sys+import time+import urllib.error+import urllib.request+from collections.abc import Sequence+from dataclasses import dataclass+from datetime import datetime, timezone+from pathlib import Path+from typing import TextIO++from benchmark_routing_algorithms import (+ BenchmarkConfig,+ DirectBaseline,+ RequiredBinary,+ positive_int,+ resolve_binaries,+ run_benchmark,+ run_checked,+)++ROUTES = (+ ("noop", "switchyard/noop"),+ ("random", "switchyard/random"),+ ("passthrough", "switchyard/passthrough"),+ ("llm_classifier", "switchyard/classifier"),+ ("stage_router", "switchyard/stage"),+)+MOCK_PORT = 8100+++@dataclass+class Child:+ """A child process and the log file that must stay open while it runs."""++ name: str+ process: subprocess.Popen[str]+ log: TextIO+ log_path: Path+++def nonnegative_int(value: str) -> int:+ """Parse a command-line integer that may be zero."""+ parsed = int(value)+ if parsed < 0:+ raise argparse.ArgumentTypeError("must be zero or greater")+ return parsed+++def tcp_port(value: str) -> int:+ """Parse a valid nonzero TCP port."""+ parsed = positive_int(value)+ if parsed > 65_535:+ raise argparse.ArgumentTypeError("must be between 1 and 65535")+ return parsed+++def parser() -> argparse.ArgumentParser:+ """Build the plain-English command-line interface."""+ command = argparse.ArgumentParser(+ description=(+ "Start the request-aware scenario backend and a local Switchyard server, send one "+ "request through every route, then run oha, AIPerf, and switchyard-soak."+ ),+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,+ )+ command.add_argument(+ "--duration",+ default="10s",+ help="time to run switchyard-soak; use an s, m, or h suffix",+ )+ command.add_argument(+ "--concurrency",+ type=positive_int,+ default=4,+ help="concurrent requests used by oha, AIPerf, and the soak run",+ )+ command.add_argument(+ "--request-count",+ type=positive_int,+ default=100,+ help="measured requests each load tool sends for each routing algorithm",+ )+ command.add_argument(+ "--mock-latency-ms",+ type=nonnegative_int,+ default=40,+ help="latency before the local scenario backend sends its first token",+ )+ command.add_argument(+ "--mock-token-latency-ms",+ type=nonnegative_int,+ default=1,+ help="latency the local scenario backend adds between streamed output tokens",+ )+ command.add_argument(+ "--server-port",+ type=tcp_port,+ default=4000,+ help="local TCP port used by switchyard-server",+ )+ command.add_argument(+ "--output-dir",+ type=Path,+ help="new directory for generated config, logs, and tool results",+ )+ return command+++def start_child(name: str, command: Sequence[str], log_path: Path) -> Child:+ """Start one long-running process and retain its combined log."""+ print(f"Starting {name}; log: {log_path}")+ log = log_path.open("w", encoding="utf-8")+ try:+ process = subprocess.Popen(+ list(command),+ stdout=log,+ stderr=subprocess.STDOUT,+ text=True,+ )+ except OSError:+ log.close()+ raise+ return Child(name=name, process=process, log=log, log_path=log_path)+++def stop_child(child: Child) -> None:+ """Stop one child and close its log, escalating to kill after five seconds."""+ if child.process.poll() is None:+ child.process.terminate()+ try:+ child.process.wait(timeout=5)+ except subprocess.TimeoutExpired:+ child.process.kill()+ child.process.wait(timeout=5)+ child.log.close()+++def wait_for_health(child: Child, url: str, expected_status: str | None = None) -> None:+ """Wait up to 30 seconds for a child process's health endpoint."""+ for _ in range(120):+ if child.process.poll() is not None:+ raise RuntimeError(+ f"{child.name} exited with status {child.process.returncode}; see {child.log_path}"+ )+ try:+ with urllib.request.urlopen(url, timeout=1) as response:+ body = response.read()+ document = json.loads(body) if expected_status is not None else None+ status = document.get("status") if isinstance(document, dict) else None+ if response.status == 200 and (+ expected_status is None or status == expected_status+ ):+ return+ except (OSError, urllib.error.URLError, json.JSONDecodeError):+ pass+ time.sleep(0.25)+ raise RuntimeError(f"{child.name} did not become healthy at {url}; see {child.log_path}")+++def check_routes(base_url: str, output_path: Path) -> None:+ """Send one HTTP request through every configured route."""+ records = []+ for _algorithm, route in ROUTES:+ body = json.dumps(+ {+ "model": route,+ "messages": [{"role": "user", "content": "Reply with exactly OK."}],+ "max_tokens": 8,+ "stream": False,+ }+ ).encode()+ request = urllib.request.Request(+ f"{base_url}/v1/chat/completions",+ data=body,+ headers={"content-type": "application/json"},+ method="POST",+ )+ with urllib.request.urlopen(request, timeout=30) as response:+ records.append(+ {+ "route": route,+ "status": response.status,+ "selected_model": response.headers.get("x-model-router-selected-model"),+ "response": json.loads(response.read()),+ }+ )+ output_path.write_text(+ "".join(f"{json.dumps(record)}\n" for record in records), encoding="utf-8"+ )+++def default_output_dir(repo_root: Path) -> Path:+ """Choose a new timestamped directory under the repository."""+ stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")+ return repo_root / "local-soak-test-results" / stamp+++def run_local_soak_test(args: argparse.Namespace) -> Path:+ """Run the complete local soak test and return its output directory."""+ repo_root = Path(__file__).resolve().parents[1]+ rust_build = (+ "Build the Rust commands with: cargo build --release -p switchyard-server "+ "-p switchyard-soak --bins --example switchyard-soak-mock"+ )+ binaries = resolve_binaries(+ (+ RequiredBinary(+ "switchyard-server",+ "SWITCHYARD_SERVER_BIN",+ os.environ.get(+ "SWITCHYARD_SERVER_BIN",+ str(repo_root / "target/release/switchyard-server"),+ ),+ rust_build,+ ),+ RequiredBinary(+ "switchyard-soak",+ "SWITCHYARD_SOAK_BIN",+ os.environ.get(+ "SWITCHYARD_SOAK_BIN", str(repo_root / "target/release/switchyard-soak")+ ),+ rust_build,+ ),+ RequiredBinary(+ "scenario backend",+ "SWITCHYARD_SOAK_MOCK_BIN",+ os.environ.get(+ "SWITCHYARD_SOAK_MOCK_BIN",+ str(repo_root / "target/release/examples/switchyard-soak-mock"),+ ),+ rust_build,+ ),+ RequiredBinary(+ "oha",+ "OHA_BIN",+ os.environ.get("OHA_BIN", "oha"),+ "Install oha with: cargo install oha",+ ),+ RequiredBinary(+ "AIPerf",+ "AIPERF_BIN",+ os.environ.get("AIPERF_BIN", "aiperf"),+ "Install AIPerf with: uv tool install --python 3.12 'aiperf==0.11.0'",+ ),+ )+ )+ server_bin = binaries["switchyard-server"]+ soak_bin = binaries["switchyard-soak"]+ mock_bin = binaries["scenario backend"]+ oha_bin = binaries["oha"]+ aiperf_bin = binaries["AIPerf"]++ output_dir = (args.output_dir or default_output_dir(repo_root)).resolve()+ output_dir.mkdir(parents=True, exist_ok=False)+ config_path = output_dir / "routes.toml"+ shutil.copy2(repo_root / "scripts/local_soak_test.toml", config_path)++ run_checked(+ "switchyard-server config validation",+ [server_bin, "--config", str(config_path), "--dry-run"],+ output_dir / "config-validation.log",+ )++ children: list[Child] = []+ try:+ mock = start_child(+ "scenario backend",+ [+ mock_bin,+ "--port",+ str(MOCK_PORT),+ "--latency-ms",+ str(args.mock_latency_ms),+ "--token-latency-ms",+ str(args.mock_token_latency_ms),+ ],+ output_dir / "scenario-backend.log",+ )+ children.append(mock)+ wait_for_health(mock, f"http://127.0.0.1:{MOCK_PORT}/health")++ server = start_child(+ "switchyard-server",+ [server_bin, "--config", str(config_path), "--port", str(args.server_port)],+ output_dir / "switchyard-server.log",+ )+ children.append(server)+ base_url = f"http://127.0.0.1:{args.server_port}"+ wait_for_health(server, f"{base_url}/health", "ok")+ check_routes(base_url, output_dir / "route-checks.jsonl")++ run_benchmark(+ BenchmarkConfig(+ base_url=base_url,+ models=ROUTES,+ concurrency=args.concurrency,+ request_count=args.request_count,+ backend_label=(+ "request-aware local backend with "+ f"{args.mock_latency_ms} ms TTFT and "+ f"{args.mock_token_latency_ms} ms per output token"+ ),+ output_dir=output_dir / "routing-benchmark",+ oha_bin=oha_bin,+ aiperf_bin=aiperf_bin,+ soak_bin=soak_bin,+ profile_runs=1,+ scenario_backend_reset_url=f"http://127.0.0.1:{MOCK_PORT}/reset",+ direct_baseline=DirectBaseline(+ base_url=f"http://127.0.0.1:{MOCK_PORT}",+ model="mock/weak",+ ),+ )+ )++ run_checked(+ "switchyard-soak API and streaming requests",+ [+ soak_bin,+ "--base-url",+ base_url,+ "--model",+ "switchyard/passthrough",+ "--duration",+ args.duration,+ "--concurrency",+ str(args.concurrency),+ "--report-interval",+ "2",+ "--invalid-canary-interval",+ "0",+ "--server-pid",+ str(server.process.pid),+ "--results-dir",+ str(output_dir / "soak"),+ ],+ output_dir / "soak.log",+ )+ finally:+ for child in reversed(children):+ stop_child(child)++ return output_dir+++def main(argv: Sequence[str] | None = None) -> int:+ """Parse arguments, run the local soak test, and print one actionable failure."""+ args = parser().parse_args(argv)+ try:+ output_dir = run_local_soak_test(args)+ except (OSError, RuntimeError) as error:+ print(f"local soak test failed: {error}", file=sys.stderr)+ return 1+ print(f"Local soak test passed; results: {output_dir}")+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/tests/test_aiperf_runner.py b/tests/test_aiperf_runner.pynew file mode 100644index 000000000..71f5fafa7--- /dev/null+++ b/tests/test_aiperf_runner.py@@ -0,0 +1,157 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++import json+import sys+import time++import pytest++import scripts.aiperf_runner as aiperf_runner+from scripts.aiperf_runner import aggregate_exports, run_profile, validate_aiperf_version+++def _write_stubborn_worker(worker) -> None:+ worker.write_text(+ """import signal+import sys+import time+from pathlib import Path++signal.signal(signal.SIGTERM, signal.SIG_IGN)+heartbeat = Path(sys.argv[1])+while True:+ heartbeat.write_text(str(time.monotonic()))+ time.sleep(0.05)+"""+ )+++def _assert_heartbeat_stopped(heartbeat) -> None:+ heartbeat_before = heartbeat.read_text()+ time.sleep(0.2)+ assert heartbeat.read_text() == heartbeat_before+++def test_run_profile_timeout_kills_the_worker_process_group(tmp_path, monkeypatch) -> None:+ fake = tmp_path / "fake_aiperf.py"+ worker = tmp_path / "worker.py"+ child_pid = tmp_path / "child-pid"+ heartbeat = tmp_path / "heartbeat"+ _write_stubborn_worker(worker)+ monkeypatch.setattr(aiperf_runner, "PROCESS_GROUP_GRACE_SECONDS", 0.2)+ fake.write_text(+ """import subprocess+import sys+import time+from pathlib import Path++child = subprocess.Popen([sys.executable, sys.argv[2], sys.argv[3]])+Path(sys.argv[1]).write_text(str(child.pid))+time.sleep(60)+"""+ )++ started_at = time.monotonic()+ with pytest.raises(RuntimeError, match="exceeded its 1-second run limit"):+ run_profile(+ [sys.executable, str(fake), str(child_pid), str(worker), str(heartbeat)],+ tmp_path / "aiperf.log",+ tmp_path / "artifacts",+ timeout_seconds=1,+ )++ assert time.monotonic() - started_at < 10+ assert child_pid.read_text().isdigit()+ _assert_heartbeat_stopped(heartbeat)+ assert "exceeded its 1-second run limit" in (tmp_path / "aiperf.log").read_text()+++def test_run_profile_stops_workers_after_the_leader_exits(tmp_path, monkeypatch) -> None:+ fake = tmp_path / "fake_aiperf.py"+ worker = tmp_path / "worker.py"+ child_pid = tmp_path / "child-pid"+ heartbeat = tmp_path / "heartbeat"+ _write_stubborn_worker(worker)+ monkeypatch.setattr(aiperf_runner, "PROCESS_GROUP_GRACE_SECONDS", 0.2)+ fake.write_text(+ """import subprocess+import sys+import time+from pathlib import Path++child = subprocess.Popen([sys.executable, sys.argv[2], sys.argv[3]])+Path(sys.argv[1]).write_text(str(child.pid))+while not Path(sys.argv[3]).exists():+ time.sleep(0.01)+"""+ )++ with pytest.raises(RuntimeError, match="did not write profile_export_aiperf.json"):+ run_profile(+ [sys.executable, str(fake), str(child_pid), str(worker), str(heartbeat)],+ tmp_path / "aiperf.log",+ tmp_path / "artifacts",+ timeout_seconds=10,+ )++ assert child_pid.read_text().isdigit()+ _assert_heartbeat_stopped(heartbeat)+++def test_process_group_probe_ignores_an_unowned_reused_group(monkeypatch) -> None:+ def deny_signal(_process_group, _signal) -> None:+ raise PermissionError++ monkeypatch.setattr(aiperf_runner.os, "killpg", deny_signal)++ assert not aiperf_runner._process_group_exists(1234)+++def test_validate_aiperf_version_rejects_uncovered_release(tmp_path) -> None:+ fake = tmp_path / "aiperf"+ fake.write_text("#!/bin/sh\nprintf '0.12.0\\n'\n")+ fake.chmod(0o755)++ with pytest.raises(RuntimeError, match="AIPerf 0.12.0 is unsupported"):+ validate_aiperf_version(str(fake))+++def test_aggregate_exports_combines_independent_runs(tmp_path) -> None:+ exports = []+ for index, throughput in enumerate((10.0, 14.0), start=1):+ path = tmp_path / f"run-{index}.json"+ document = {+ "aiperf_version": "0.11.0",+ "error_request_count": None,+ "request_count": {"unit": "requests", "avg": 20},+ "request_throughput": {+ "unit": "requests/sec",+ "avg": throughput,+ },+ "time_to_first_token": {"unit": "ms", "p50": 10.0},+ }+ if index == 2:+ del document["error_request_count"]+ document["error_summary"] = []+ path.write_text(json.dumps(document))+ exports.append(path)++ aggregate_path = aggregate_exports(exports, tmp_path / "aggregate" / "report.json")+ metrics = json.loads(aggregate_path.read_text())["metrics"]++ assert metrics["request_throughput_avg"]["mean"] == 12.0+ assert metrics["request_throughput_avg"]["ci_low"] < 0+ assert metrics["error_request_count_avg"]["mean"] == 0.0++ mismatched = json.loads(exports[1].read_text())+ mismatched["time_to_first_token"]["unit"] = "seconds"+ exports[1].write_text(json.dumps(mismatched))+ with pytest.raises(RuntimeError, match="disagree on the unit"):+ aggregate_exports(exports, tmp_path / "invalid" / "report.json")++ mismatched["time_to_first_token"]["unit"] = "ms"+ mismatched["request_count"]["avg"] = True+ exports[1].write_text(json.dumps(mismatched))+ with pytest.raises(RuntimeError, match="has no numeric request_count.avg"):+ aggregate_exports(exports, tmp_path / "malformed" / "report.json")diff --git a/tests/test_routing_overhead_plot.py b/tests/test_routing_overhead_plot.pynew file mode 100644index 000000000..9a7ddf42e--- /dev/null+++ b/tests/test_routing_overhead_plot.py@@ -0,0 +1,64 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++import xml.etree.ElementTree as ET++from scripts.routing_overhead_plot import OverheadPlotRow, write_overhead_plot+++def test_overhead_plot_preserves_matrix_and_unavailable_values(tmp_path) -> None:+ path = tmp_path / "routing-overhead.svg"+ rows = (+ OverheadPlotRow("short-<interactive>", "fixed", "random", 3.0, 25.0, -80.0, -11.43),+ OverheadPlotRow("short-<interactive>", "fixed", "stage&router", None, None, -14.0, -2.0),+ OverheadPlotRow("long-context", "traffic-burst", "random", 1.25, 3.0, None, None),+ OverheadPlotRow("long-context", "traffic-burst", "stage&router", -0.5, -1.0, 7.0, 1.0),+ )++ write_overhead_plot(path, rows)++ root = ET.fromstring(path.read_text())+ namespace = {"svg": "http://www.w3.org/2000/svg"}+ cells = root.findall(".//svg:g", namespace)+ text_nodes = root.findall(".//svg:text", namespace)+ values = {+ (+ cell.attrib["data-scenario"],+ cell.attrib["data-load"],+ cell.attrib["data-route"],+ cell.attrib["data-metric"],+ ): " ".join("".join(cell.itertext()).split())+ for cell in cells+ }+ assert len(values) == 8+ assert (+ values[+ (+ "short-<interactive>",+ "fixed",+ "random",+ "output_tokens_per_second_delta",+ )+ ]+ == "short-<interactive>, fixed, random: -80.00 tokens/s (-11.43%) "+ "-80.00 tokens/s (-11.43%)"+ )+ assert (+ "n/a"+ in values[("long-context", "traffic-burst", "random", "output_tokens_per_second_delta")]+ )+ assert "n/a" in values[("short-<interactive>", "fixed", "stage&router", "ttft_p50_delta_ms")]+ centered_labels = {+ node.text: node.attrib["text-anchor"]+ for node in text_nodes+ if node.text in {"Routing overhead versus the direct backend", "short <interactive>"}+ }+ assert centered_labels == {+ "Routing overhead versus the direct backend": "middle",+ "short <interactive>": "middle",+ }+ assert all(+ text.attrib["text-anchor"] == "middle"+ for cell in cells+ for text in cell.findall("svg:text", namespace)+ )diff --git a/tests/test_routing_performance_report.py b/tests/test_routing_performance_report.pynew file mode 100644index 000000000..445357cc1--- /dev/null+++ b/tests/test_routing_performance_report.py@@ -0,0 +1,391 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++import csv+import json+from dataclasses import replace+from pathlib import Path++import pytest++import scripts.benchmark_routing_algorithms as benchmark+from scripts.benchmark_routing_algorithms import (+ BenchmarkArm,+ BenchmarkConfig,+ DirectBaseline,+ ScenarioDefinition,+ compare_to_direct_backend,+ materialize_aiperf_input,+ parse_result,+ profile_request_count,+ write_report,+)+++def test_report_combines_scenarios_single_and_aggregate_results(tmp_path) -> None:+ oha_path = tmp_path / "oha.json"+ single_path = tmp_path / "profile_export_aiperf.json"+ direct_path = tmp_path / "profile_export_aiperf_direct.json"+ aggregate_path = tmp_path / "profile_export_aiperf_aggregate.json"+ oha_path.write_text(+ json.dumps(+ {+ "summary": {"successRate": 1.0, "requestsPerSec": 120.5},+ "latencyPercentiles": {"p50": 0.012, "p99": 0.045},+ }+ )+ )+ single_path.write_text(+ json.dumps(+ {+ "error_request_count": {"unit": "requests", "avg": 0},+ "request_count": {"unit": "requests", "avg": 4},+ "request_throughput": {"unit": "requests/sec", "avg": 95.5},+ "request_latency": {"unit": "ms", "p50": 40.0},+ "time_to_first_token": {"unit": "ms", "p50": 15.0, "p99": 35.0},+ "output_token_throughput": {"unit": "tokens/sec", "avg": 620.0},+ "e2e_output_token_throughput": {+ "unit": "tokens/sec/user",+ "p50": 200.0,+ },+ }+ )+ )+ direct_path.write_text(+ json.dumps(+ {+ "error_request_count": {"unit": "requests", "avg": 0},+ "request_count": {"unit": "requests", "avg": 4},+ "request_throughput": {"unit": "requests/sec", "avg": 100.0},+ "request_latency": {"unit": "ms", "p50": 35.0},+ "time_to_first_token": {"unit": "ms", "p50": 12.0, "p99": 30.0},+ "output_token_throughput": {"unit": "tokens/sec", "avg": 700.0},+ }+ )+ )+ aggregate_path.write_text(+ json.dumps(+ {+ "metrics": {+ "error_request_count_avg": {"unit": "requests", "mean": 2},+ "request_count_avg": {"unit": "requests", "mean": 4},+ "request_throughput_avg": {+ "unit": "requests/sec",+ "mean": 80.0,+ "cv": 0.03,+ "ci_low": 77.0,+ "ci_high": 83.0,+ },+ "request_latency_p50": {"unit": "seconds", "mean": 0.05},+ "time_to_first_token_p50": {"unit": "ms", "mean": 20.0},+ "time_to_first_token_p99": {"unit": "ms", "mean": 45.0},+ "output_token_throughput_avg": {+ "unit": "tokens/sec",+ "mean": 500.0,+ },+ }+ }+ )+ )+ short = ScenarioDefinition(+ id="short-interactive",+ group="core",+ description="short baseline",+ expected="all requests succeed",+ expected_error_rate_min=0.0,+ expected_error_rate_max=0.0,+ input_file=tmp_path / "short.json",+ load_profiles=(),+ )+ failure = ScenarioDefinition(+ id="failure-pressure",+ group="resilience",+ description="failure injection",+ expected="failures stay bounded",+ expected_error_rate_min=0.01,+ expected_error_rate_max=0.75,+ input_file=tmp_path / "failure.json",+ load_profiles=(),+ )+ before = {+ "models": {+ "mock/weak": {"calls": 10, "errors": 0},+ "mock/classifier": {"calls": 3, "errors": 0},+ },+ "classifier": {+ "total_requests": 3,+ "total_errors": 0,+ "models": {+ "mock/classifier": {+ "calls": 3,+ "errors": 0,+ "model_call_latency": {"count": 3, "total_ms": 6},+ }+ },+ },+ "routing_overhead": {"count": 10, "total_ms": 20},+ }+ after = {+ "models": {+ "mock/weak": {"calls": 20, "errors": 2},+ "mock/strong": {"calls": 2, "errors": 0},+ "mock/classifier": {"calls": 5, "errors": 1},+ },+ "classifier": {+ "total_requests": 5,+ "total_errors": 1,+ "models": {+ "mock/classifier": {+ "calls": 5,+ "errors": 1,+ "model_call_latency": {"count": 5, "total_ms": 14},+ }+ },+ },+ "routing_overhead": {"count": 22, "total_ms": 50},+ }+ results = compare_to_direct_backend(+ (+ parse_result(+ BenchmarkArm(+ "direct-backend",+ "mock/weak",+ "http://127.0.0.1:8100",+ True,+ ),+ short,+ "fixed",+ None,+ direct_path,+ {},+ {},+ ),+ parse_result(+ BenchmarkArm(+ "random",+ "switchyard/random",+ "http://127.0.0.1:4000",+ False,+ ),+ short,+ "fixed",+ oha_path,+ single_path,+ before,+ after,+ ),+ parse_result(+ BenchmarkArm(+ "classifier",+ "switchyard/classifier",+ "http://127.0.0.1:4000",+ False,+ ),+ failure,+ "fixed",+ None,+ aggregate_path,+ before,+ after,+ ),+ )+ )+ output_dir = tmp_path / "report"+ output_dir.mkdir()+ config = BenchmarkConfig(+ base_url="http://127.0.0.1:4000",+ models=(("random", "switchyard/random"),),+ concurrency=100,+ request_count=1000,+ backend_label="test backend",+ output_dir=output_dir,+ oha_bin="oha",+ aiperf_bin="aiperf",+ soak_bin="switchyard-soak",+ direct_baseline=DirectBaseline("http://127.0.0.1:8100", "mock/weak"),+ )++ write_report(config, results)++ report = (output_dir / "report.md").read_text()+ assert "## Routing overhead versus the direct backend" in report+ assert "" in report+ assert (+ report.index("## Routing overhead versus the direct backend")+ < report.index("## Run details")+ < report.index("## Throughput and latency")+ )+ assert (+ "| short interactive | random | fixed | +5.00 ms | +3.00 ms | +25.00% | "+ "-4.50% | -80.00 tokens/s | -11.43% |" in report+ )+ assert "| short interactive | short baseline |" in report+ assert "| random | short-interactive | fixed | 120.50 | 95.50 |" in report+ assert "## Repeatability" in report+ assert "## Resilience" in report+ assert (+ "| classifier | failure-pressure | fixed | 50.00% | 1.00%–75.00% | PASS | `{"+ '"mock/weak":2}` | failures stay bounded |' in report+ )+ rows = list(csv.DictReader((output_dir / "report.csv").open()))+ rows_by_key = {(row["algorithm"], row["scenario"]): row for row in rows}+ random_row = rows_by_key[("random", "short-interactive")]+ assert random_row["selected_model_calls"] == '{"mock/strong":2,"mock/weak":10}'+ assert random_row["selected_model_share"] == '{"mock/strong":0.1667,"mock/weak":0.8333}'+ assert random_row["selected_model_errors"] == '{"mock/weak":2}'+ payload = json.loads((output_dir / "report.json").read_text())+ payload_by_key = {(row["algorithm"], row["scenario"]): row for row in payload["results"]}+ direct_row = payload_by_key[("direct-backend", "short-interactive")]+ random_payload = payload_by_key[("random", "short-interactive")]+ classifier_payload = payload_by_key[("classifier", "failure-pressure")]+ assert direct_row["aiperf_itl_p50_ms"] is None+ assert direct_row["aiperf_request_throughput_delta_pct"] is None+ assert random_payload["aiperf_output_tokens_per_second_per_user_p50"] == 200.0+ assert random_payload["aiperf_ttft_p50_delta_pct"] == 25.0+ assert random_payload["aiperf_output_tokens_per_second_delta"] == -80.0+ assert random_payload["scenario_description"] == "short baseline"+ assert classifier_payload["aiperf_request_latency_p50_ms"] == 50.0+ assert classifier_payload["aiperf_request_throughput_cv"] == 0.03+ assert classifier_payload["classifier_latency_avg_ms"] == 4.0++ assert (output_dir / "routing-overhead.svg").is_file()++ with pytest.raises(RuntimeError, match="missing direct baseline for mixed-traffic fixed"):+ compare_to_direct_backend((results[0], replace(results[1], scenario="mixed-traffic")))+++def test_materialized_sessions_are_unique_and_cover_request_count(tmp_path) -> None:+ scenario_path = tmp_path / "short.json"+ scenario_path.write_text(+ json.dumps(+ {+ "data": [+ {+ "session_id": "short-0",+ "payloads": [{"model": "test"}, {"model": "test"}],+ }+ ]+ }+ )+ )+ scenario = ScenarioDefinition(+ id="short-interactive",+ group="core",+ description="short baseline",+ expected="all requests succeed",+ expected_error_rate_min=0.0,+ expected_error_rate_max=0.0,+ input_file=scenario_path,+ load_profiles=(),+ )++ expanded_input = materialize_aiperf_input(scenario, tmp_path / "expanded.json", 5)+ expanded_sessions = json.loads(expanded_input.read_text())["data"]++ assert len(expanded_sessions) == 3+ assert len({session["session_id"] for session in expanded_sessions}) == 3+++def test_traffic_burst_request_count_integrates_rate_series(tmp_path) -> None:+ config = BenchmarkConfig(+ base_url="http://127.0.0.1:4000",+ models=(("random", "switchyard/random"),),+ concurrency=2,+ request_count=100,+ backend_label="test backend",+ output_dir=tmp_path,+ oha_bin="oha",+ aiperf_bin="aiperf",+ soak_bin="switchyard-soak",+ )+ profile = {+ "kind": "traffic_burst",+ "duration_seconds": 30,+ "points": [+ {"time_s": 0, "rate_multiplier": 1},+ {"time_s": 10, "rate_multiplier": 1},+ {"time_s": 11, "rate_multiplier": 10},+ {"time_s": 16, "rate_multiplier": 10},+ {"time_s": 17, "rate_multiplier": 1},+ {"time_s": 30, "rate_multiplier": 1},+ ],+ }++ assert profile_request_count(config, profile) == 168+++def test_aiperf_cells_use_disjoint_artifacts(tmp_path, monkeypatch) -> None:+ scenario_path = tmp_path / "short.json"+ scenario_path.write_text(+ json.dumps(+ {+ "data": [+ {+ "session_id": "short-0",+ "payloads": [{"model": "switchyard/random"}],+ }+ ]+ }+ )+ )+ scenario = ScenarioDefinition(+ id="short-interactive",+ group="core",+ description="short baseline",+ expected="all requests succeed",+ expected_error_rate_min=0.0,+ expected_error_rate_max=0.0,+ input_file=scenario_path,+ load_profiles=(),+ )+ config = BenchmarkConfig(+ base_url="http://127.0.0.1:4000",+ models=(("random", "switchyard/random"),),+ concurrency=1,+ request_count=1,+ backend_label="test backend",+ output_dir=tmp_path,+ oha_bin="oha",+ aiperf_bin="aiperf",+ soak_bin="switchyard-soak",+ profile_runs=1,+ )+ arm = BenchmarkArm(+ "random",+ "switchyard/random",+ "http://127.0.0.1:4000",+ False,+ )+ observed: list[tuple[Path, Path]] = []++ def fake_run_profile(+ _command, log_path: Path, artifact_dir: Path, _timeout_seconds: int+ ) -> Path:+ observed.append((log_path, artifact_dir))+ artifact_dir.mkdir(parents=True)+ export = artifact_dir / "profile_export_aiperf.json"+ export.write_text("{}")+ return export++ monkeypatch.setattr(benchmark, "run_profile", fake_run_profile)++ first = benchmark.run_aiperf(+ config,+ arm,+ scenario,+ {"kind": "fixed"},+ tmp_path,+ "algorithm-00-short-interactive-fixed",+ )+ second = benchmark.run_aiperf(+ config,+ arm,+ scenario,+ {"kind": "fixed"},+ tmp_path,+ "algorithm-00-long-context-fixed",+ )++ assert first != second+ assert observed[0][0] != observed[1][0]+ assert observed[0][1] != observed[1][1]