Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
9.3 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Request-aware local backend used by `scripts/run_local_soak_test.py`.

use std::collections::HashMap;
use std::convert::Infallible;
use std::process::ExitCode;
use std::sync::Arc;
use std::time::Duration;

use axum::Router;
use axum::extract::{Json, State};
use axum::http::StatusCode;
use axum::response::sse::{Event, Sse};
use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
use clap::Parser;
use futures_util::{StreamExt, stream};
use parking_lot::Mutex;
use serde_json::{Value, json};

#[derive(Parser)]
#[command(
    name = "switchyard-soak-mock",
    about = "Start the request-aware backend for the local Switchyard scenario test",
    after_long_help = "Example:\n  cargo run --release -p switchyard-soak --example switchyard-soak-mock -- --port 8100 --latency-ms 40",
    version
)]
struct Args {
    /// Local TCP port used by the backend.
    #[arg(long, default_value_t = 8100)]
    port: u16,

    /// Artificial delay, in milliseconds, added to ordinary responses.
    #[arg(long, default_value_t = 40)]
    latency_ms: u64,

    /// Artificial delay between streamed output tokens, in milliseconds.
    #[arg(long, default_value_t = 1)]
    token_latency_ms: u64,
}

#[derive(Clone)]
struct BackendState {
    latency: Duration,
    token_latency: Duration,
    attempts: Arc<Mutex<HashMap<String, u64>>>,
}

// AIPerf requires this OpenAI field; a fixed value keeps local runs deterministic.
const FIXED_CREATED_AT: u64 = 1_700_000_000;

fn scenario_marker(body: &Value) -> Option<&str> {
    body.get("messages")?
        .as_array()?
        .iter()
        .filter_map(|message| message.get("content").and_then(Value::as_str))
        .find_map(|content| {
            let marker = content.split_once("[scenario:")?.1;
            marker.split_once(']').map(|(name, _rest)| name)
        })
}

fn classifier_content(marker: Option<&str>) -> &'static str {
    if marker == Some("classifier_invalid") {
        return "not a JSON verdict";
    }
    if marker == Some("classifier_hard") {
        return r#"{"crux":"distributed diagnosis","primary_rule":"LIM-1","capability_boundary":"unsupported","p_solve":0.1}"#;
    }
    r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":1.0}"#
}

fn should_inject_failure(name: &str, attempt: u64) -> bool {
    name == "upstream_500" || attempt <= 2
}

fn completion(model: &str, content: &str) -> Value {
    json!({
        "id": "chatcmpl-switchyard-soak",
        "object": "chat.completion",
        "created": FIXED_CREATED_AT,
        "model": model,
        "choices": [{
            "index": 0,
            "message": {"role": "assistant", "content": content},
            "finish_reason": "stop"
        }],
        "usage": {"prompt_tokens": 32, "completion_tokens": 2, "total_tokens": 34}
    })
}

fn requested_output_tokens(body: &Value) -> u64 {
    body.get("max_completion_tokens")
        .or_else(|| body.get("max_tokens"))
        .and_then(Value::as_u64)
        .unwrap_or(2)
        .clamp(1, 4_096)
}

fn stream(
    model: &str,
    truncated: bool,
    completion_tokens: u64,
    token_latency: Duration,
) -> Response {
    let mut events = Vec::with_capacity(completion_tokens as usize + 2);
    for index in 0..completion_tokens {
        events.push(
            json!({
                "id": "chatcmpl-switchyard-soak",
                "object": "chat.completion.chunk",
                "created": FIXED_CREATED_AT,
                "model": model,
                "choices": [{
                    "index": 0,
                    "delta": {
                        "role": (index == 0).then_some("assistant"),
                        "content": if index == 0 { "token" } else { " token" }
                    },
                    "finish_reason": null
                }]
            })
            .to_string(),
        );
        if truncated {
            break;
        }
    }
    if !truncated {
        events.push(
            json!({
                "id": "chatcmpl-switchyard-soak",
                "object": "chat.completion.chunk",
                "created": FIXED_CREATED_AT,
                "model": model,
                "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
                "usage": {
                    "prompt_tokens": 32,
                    "completion_tokens": completion_tokens,
                    "total_tokens": 32 + completion_tokens
                }
            })
            .to_string(),
        );
        events.push("[DONE]".to_string());
    }
    let events =
        stream::iter(events.into_iter().enumerate()).then(move |(index, data)| async move {
            if index > 0 {
                tokio::time::sleep(token_latency).await;
            }
            Ok::<Event, Infallible>(Event::default().data(data))
        });
    Sse::new(events).into_response()
}

async fn chat(State(state): State<BackendState>, Json(body): Json<Value>) -> Response {
    let model = body
        .get("model")
        .and_then(Value::as_str)
        .unwrap_or_default();
    let marker = scenario_marker(&body);
    if marker == Some("client_cancellation") {
        tokio::time::sleep(Duration::from_secs(2)).await;
    } else {
        tokio::time::sleep(state.latency).await;
    }

    if model == "mock/classifier" {
        return Json(completion(model, classifier_content(marker))).into_response();
    }
    if model == "mock/weak" && marker == Some("context_overflow") {
        return (
            StatusCode::BAD_REQUEST,
            Json(json!({
                "error": {
                    "code": "context_length_exceeded",
                    "message": "the weak target context window is too small"
                }
            })),
        )
            .into_response();
    }

    for (name, status) in [
        ("upstream_429", StatusCode::TOO_MANY_REQUESTS),
        ("upstream_500", StatusCode::INTERNAL_SERVER_ERROR),
    ] {
        if marker == Some(name) {
            let key = format!("{model}:{name}");
            let attempt = {
                let mut attempts = state.attempts.lock();
                let attempt = attempts.entry(key).or_default();
                *attempt += 1;
                *attempt
            };
            if should_inject_failure(name, attempt) {
                // Keep retry-exhaustion scenarios fast enough for local benchmarks.
                return (
                    status,
                    [("retry-after", "0")],
                    Json(
                        json!({"error": {"message": format!("injected {name} attempt {attempt}")}}),
                    ),
                )
                    .into_response();
            }
        }
    }

    if body.get("stream") == Some(&Value::Bool(true)) {
        return stream(
            model,
            marker == Some("truncated_stream"),
            requested_output_tokens(&body),
            state.token_latency,
        );
    }
    Json(completion(model, "OK")).into_response()
}

async fn run(args: Args) -> Result<(), String> {
    if args.port == 0 {
        return Err("--port must be greater than zero".to_string());
    }
    let listener = tokio::net::TcpListener::bind(("127.0.0.1", args.port))
        .await
        .map_err(|error| error.to_string())?;
    let address = listener.local_addr().map_err(|error| error.to_string())?;
    let state = BackendState {
        latency: Duration::from_millis(args.latency_ms),
        token_latency: Duration::from_millis(args.token_latency_ms),
        attempts: Arc::new(Mutex::new(HashMap::new())),
    };
    let app = Router::new()
        .route("/health", get(|| async { Json(json!({"status": "ok"})) }))
        .route(
            "/reset",
            post(|State(state): State<BackendState>| async move {
                state.attempts.lock().clear();
                Json(json!({"status": "reset"}))
            }),
        )
        .route("/v1/chat/completions", post(chat))
        .with_state(state);
    println!("Scenario backend is ready at http://{address}");
    axum::serve(listener, app)
        .with_graceful_shutdown(async {
            if let Err(error) = tokio::signal::ctrl_c().await {
                eprintln!("could not register the shutdown signal: {error}");
                std::future::pending::<()>().await;
            }
        })
        .await
        .map_err(|error| error.to_string())
}

#[cfg(test)]
mod tests {
    use serde_json::json;

    use super::{requested_output_tokens, should_inject_failure};

    #[test]
    fn output_tokens_follow_openai_limits_and_stay_bounded() {
        assert_eq!(requested_output_tokens(&json!({"max_tokens": 512})), 512);
        assert_eq!(
            requested_output_tokens(&json!({"max_completion_tokens": 8_192})),
            4_096
        );
        assert_eq!(requested_output_tokens(&json!({})), 2);
    }

    #[test]
    fn failure_pressure_recovers_429_but_not_500() {
        assert!(should_inject_failure("upstream_429", 2));
        assert!(!should_inject_failure("upstream_429", 3));
        assert!(should_inject_failure("upstream_500", 11));
    }
}

#[tokio::main]
async fn main() -> ExitCode {
    match run(Args::parse()).await {
        Ok(()) => ExitCode::SUCCESS,
        Err(error) => {
            eprintln!("switchyard-soak-mock failed: {error}");
            ExitCode::FAILURE
        }
    }
}