Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
19.3 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! HTTP against the public Switchyard APIs: preflight, one request, and server-state reads.

use futures_util::StreamExt;
use reqwest::Client;
use reqwest::header::CONTENT_TYPE;
use serde_json::{Value, json};

const SERVER_REQUESTS_METRIC: &str = "switchyard_total_requests";
const SERVER_ERRORS_METRIC: &str = "switchyard_total_errors";

/// One public Switchyard API the soak test exercises.
#[derive(Clone, Copy, PartialEq, Eq)]
pub enum Endpoint {
    Chat,
    Messages,
    Responses,
}

impl Endpoint {
    pub const ALL: [Endpoint; 3] = [Endpoint::Chat, Endpoint::Messages, Endpoint::Responses];

    pub fn path(self) -> &'static str {
        match self {
            Endpoint::Chat => "/v1/chat/completions",
            Endpoint::Messages => "/v1/messages",
            Endpoint::Responses => "/v1/responses",
        }
    }

    /// Field a successful response for this endpoint must contain.
    fn required_field(self) -> &'static str {
        match self {
            Endpoint::Chat => "choices",
            Endpoint::Messages => "content",
            Endpoint::Responses => "output",
        }
    }

    pub fn as_str(self) -> &'static str {
        match self {
            Endpoint::Chat => "chat",
            Endpoint::Messages => "messages",
            Endpoint::Responses => "responses",
        }
    }
}

/// Keep at most *limit* characters, on a char boundary, for a logged detail string.
fn truncate(text: &str, limit: usize) -> String {
    text.chars().take(limit).collect()
}

fn transport_error_kind(error: &reqwest::Error) -> &'static str {
    if error.is_timeout() {
        "timeout"
    } else if error.is_decode() {
        "request_error"
    } else if error.is_connect() || error.is_request() || error.is_body() {
        "transport"
    } else {
        "request_error"
    }
}

/// Build one request body for a public Switchyard API.
pub fn request_body(
    endpoint: Endpoint,
    model: &str,
    prompt: &str,
    max_output_tokens: u32,
    stream: bool,
) -> Value {
    match endpoint {
        // Chat Completions and Anthropic Messages take the same model/messages/max_tokens body.
        Endpoint::Chat | Endpoint::Messages => json!({
            "model": model,
            "messages": [{"role": "user", "content": prompt}],
            "max_tokens": max_output_tokens,
            "temperature": 0,
            "stream": stream,
        }),
        Endpoint::Responses => json!({
            "model": model,
            "input": prompt,
            "max_output_tokens": max_output_tokens,
            "stream": stream,
        }),
    }
}

#[derive(Default)]
struct Metrics {
    requests: Option<f64>,
    errors: Option<f64>,
}

fn parse_metrics(text: &str) -> Metrics {
    let mut metrics = Metrics::default();
    for line in text.lines() {
        if line.is_empty() || line.starts_with('#') {
            continue;
        }
        let Some((name_part, rest)) = line.split_once(' ') else {
            continue;
        };
        let name = name_part.split('{').next().unwrap_or(name_part);
        if let Some(token) = rest.split_whitespace().next()
            && let Ok(value) = token.parse::<f64>()
        {
            match name {
                SERVER_REQUESTS_METRIC => metrics.requests = Some(value),
                SERVER_ERRORS_METRIC => metrics.errors = Some(value),
                _ => {}
            }
        }
    }
    metrics
}

#[derive(Debug)]
pub struct RequestError {
    pub kind: String,
    pub detail: String,
}

impl RequestError {
    fn new(kind: impl Into<String>, detail: impl Into<String>) -> Self {
        Self {
            kind: kind.into(),
            detail: detail.into(),
        }
    }

    fn transport(error: &reqwest::Error) -> Self {
        Self::new(
            transport_error_kind(error),
            truncate(&error.to_string(), 500),
        )
    }
}

struct StreamValidator {
    endpoint: Endpoint,
    event_name: Option<String>,
    saw_json: bool,
    saw_terminal: bool,
}

impl StreamValidator {
    fn new(endpoint: Endpoint) -> Self {
        Self {
            endpoint,
            event_name: None,
            saw_json: false,
            saw_terminal: false,
        }
    }

    fn read_line(&mut self, line: &[u8]) -> Result<(), RequestError> {
        let line = line.strip_suffix(b"\r").unwrap_or(line);
        if line.is_empty() {
            self.event_name = None;
            return Ok(());
        }
        let line = std::str::from_utf8(line)
            .map_err(|error| RequestError::new("invalid_stream", error.to_string()))?;
        if let Some(event_name) = line.strip_prefix("event:") {
            self.event_name = Some(event_name.trim_start().to_string());
            return Ok(());
        }
        let Some(data) = line.strip_prefix("data:") else {
            return Ok(());
        };
        let data = data.strip_prefix(' ').unwrap_or(data);
        if self.saw_terminal {
            return Err(RequestError::new(
                "invalid_stream",
                "stream contained data after its terminal event",
            ));
        }
        if data == "[DONE]" {
            if self.endpoint != Endpoint::Chat {
                return Err(RequestError::new(
                    "invalid_stream",
                    format!(
                        "{} stream contained an OpenAI [DONE] marker",
                        self.endpoint.as_str()
                    ),
                ));
            }
            self.saw_terminal = true;
            return Ok(());
        }

        let payload: Value = serde_json::from_str(data).map_err(|error| {
            RequestError::new("invalid_stream", format!("invalid SSE JSON: {error}"))
        })?;
        let event_type = payload.get("type").and_then(Value::as_str);
        if self.event_name.as_deref() == Some("error")
            || event_type == Some("error")
            || payload.get("error").is_some_and(|error| !error.is_null())
        {
            let detail = payload
                .pointer("/error/message")
                .or_else(|| payload.get("message"))
                .and_then(Value::as_str)
                .unwrap_or("stream returned an error event");
            return Err(RequestError::new("stream_error", truncate(detail, 500)));
        }

        match self.endpoint {
            Endpoint::Chat => {}
            Endpoint::Messages | Endpoint::Responses => {
                let event_type = event_type.ok_or_else(|| {
                    RequestError::new("invalid_stream", "SSE data did not contain an event type")
                })?;
                if self.event_name.as_deref() != Some(event_type) {
                    return Err(RequestError::new(
                        "invalid_stream",
                        format!(
                            "SSE event name {:?} did not match data type {event_type:?}",
                            self.event_name
                        ),
                    ));
                }
                self.saw_terminal = matches!(
                    (self.endpoint, event_type),
                    (Endpoint::Messages, "message_stop")
                        | (
                            Endpoint::Responses,
                            "response.completed" | "response.incomplete"
                        )
                );
            }
        }
        self.saw_json = true;
        Ok(())
    }

    fn finish(self) -> Result<(), RequestError> {
        if !self.saw_json {
            return Err(RequestError::new(
                "empty_stream",
                "successful streaming response contained no JSON events",
            ));
        }
        if !self.saw_terminal {
            return Err(RequestError::new(
                "invalid_stream",
                format!(
                    "{} stream ended without its terminal event",
                    self.endpoint.as_str()
                ),
            ));
        }
        Ok(())
    }
}

/// Send one request and validate its response body or event stream.
pub async fn send_request(
    client: &Client,
    base_url: &str,
    endpoint: Endpoint,
    session_id: &str,
    body: &Value,
) -> Result<(), RequestError> {
    let url = format!("{base_url}{}", endpoint.path());
    let stream = body.get("stream").and_then(Value::as_bool).unwrap_or(false);
    let response = match client
        .post(&url)
        .header("x-switchyard-session-id", session_id)
        .json(body)
        .send()
        .await
    {
        Ok(response) => response,
        Err(error) => return Err(RequestError::transport(&error)),
    };

    if stream {
        let status = response.status();
        if !status.is_success() {
            let content = response.text().await.unwrap_or_default();
            return Err(RequestError::new(
                format!("http_{}", status.as_u16()),
                truncate(&content, 500),
            ));
        }
        let content_type = response
            .headers()
            .get(CONTENT_TYPE)
            .and_then(|value| value.to_str().ok())
            .unwrap_or("")
            .to_string();
        if !content_type.contains("text/event-stream") {
            let content = response.text().await.unwrap_or_default();
            return Err(RequestError::new(
                "invalid_stream",
                format!(
                    "expected text/event-stream, received {content_type:?}: {}",
                    truncate(&content, 300)
                ),
            ));
        }
        let mut bytes = response.bytes_stream();
        let mut pending = Vec::new();
        let mut validator = StreamValidator::new(endpoint);
        while let Some(chunk) = bytes.next().await {
            match chunk {
                Ok(chunk) => {
                    pending.extend_from_slice(&chunk);
                    while let Some(newline) = pending.iter().position(|byte| *byte == b'\n') {
                        let mut line = pending.drain(..=newline).collect::<Vec<_>>();
                        line.pop();
                        validator.read_line(&line)?;
                    }
                }
                Err(error) => {
                    return Err(RequestError::transport(&error));
                }
            }
        }
        if !pending.is_empty() {
            validator.read_line(&pending)?;
        }
        return validator.finish();
    }

    let status = response.status();
    if !status.is_success() {
        let content = response.text().await.unwrap_or_default();
        return Err(RequestError::new(
            format!("http_{}", status.as_u16()),
            truncate(&content, 500),
        ));
    }
    let text = match response.text().await {
        Ok(text) => text,
        Err(error) => return Err(RequestError::transport(&error)),
    };
    let payload: Value = match serde_json::from_str(&text) {
        Ok(payload) => payload,
        Err(error) => return Err(RequestError::new("invalid_json", error.to_string())),
    };
    if !payload.is_object() || payload.get("error").is_some_and(|error| !error.is_null()) {
        return Err(RequestError::new("invalid_response", truncate(&text, 500)));
    }
    let field = endpoint.required_field();
    if payload.get(field).is_none() {
        return Err(RequestError::new(
            "invalid_response",
            format!(
                "successful {} response did not contain {field:?}: {}",
                endpoint.as_str(),
                truncate(&text, 300)
            ),
        ));
    }
    Ok(())
}

/// Check liveness and verify that the requested model is advertised.
pub async fn preflight(client: &Client, base_url: &str, model: &str) -> Result<(), String> {
    let health = client
        .get(format!("{base_url}/health"))
        .send()
        .await
        .map_err(|error| error.to_string())?
        .error_for_status()
        .map_err(|error| error.to_string())?;
    let health_text = health.text().await.map_err(|error| error.to_string())?;
    let health_body: Value = serde_json::from_str(&health_text).map_err(|_| {
        format!(
            "GET /health did not return JSON: {}",
            truncate(&health_text, 300)
        )
    })?;
    if health_body.get("status").and_then(Value::as_str) != Some("ok") {
        return Err(format!(
            "GET /health returned an unexpected body: {}",
            truncate(&health_text, 300)
        ));
    }

    let response = client
        .get(format!("{base_url}/v1/models"))
        .send()
        .await
        .map_err(|error| error.to_string())?
        .error_for_status()
        .map_err(|error| error.to_string())?;
    let text = response.text().await.map_err(|error| error.to_string())?;
    let body: Value = serde_json::from_str(&text).map_err(|_| {
        format!(
            "GET /v1/models did not return JSON: {}",
            truncate(&text, 300)
        )
    })?;
    let entries = body.get("data").and_then(Value::as_array).ok_or_else(|| {
        format!(
            "GET /v1/models returned an unexpected body: {}",
            truncate(&text, 300)
        )
    })?;
    if !entries
        .iter()
        .any(|entry| entry.get("id").and_then(Value::as_str) == Some(model))
    {
        return Err(format!("model {model:?} is not listed by GET /v1/models"));
    }
    Ok(())
}

pub struct ServerState {
    pub healthy: bool,
    pub requests: Option<f64>,
    pub errors: Option<f64>,
}

/// Read liveness and cumulative metrics; one bad read becomes one failed sample.
pub async fn read_server_state(client: &Client, base_url: &str) -> ServerState {
    let healthy = match client.get(format!("{base_url}/health")).send().await {
        Ok(response) if response.status() == reqwest::StatusCode::OK => response
            .text()
            .await
            .ok()
            .and_then(|text| serde_json::from_str::<Value>(&text).ok())
            .and_then(|body| {
                body.get("status")
                    .and_then(Value::as_str)
                    .map(|s| s == "ok")
            })
            .unwrap_or(false),
        _ => false,
    };
    let metrics = match client.get(format!("{base_url}/metrics")).send().await {
        Ok(response) if response.status().is_success() => response
            .text()
            .await
            .map(|text| parse_metrics(&text))
            .unwrap_or_default(),
        _ => Metrics::default(),
    };
    ServerState {
        healthy,
        requests: metrics.requests,
        errors: metrics.errors,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn buffered_responses_accept_only_absent_or_null_errors() {
        let client = Client::builder()
            .no_proxy()
            .timeout(std::time::Duration::from_secs(5))
            .build()
            .unwrap();
        for (payload, should_pass) in [
            (json!({"output": [], "status": "completed"}), true),
            (
                json!({"output": [], "status": "completed", "error": null}),
                true,
            ),
            (json!({"output": [], "error": {"message": "boom"}}), false),
        ] {
            let app = axum::Router::new().route(
                "/v1/responses",
                axum::routing::post(async move || axum::Json(payload)),
            );
            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
            let base_url = format!("http://{}", listener.local_addr().unwrap());
            let server = tokio::spawn(async move { axum::serve(listener, app).await });
            let result = send_request(
                &client,
                &base_url,
                Endpoint::Responses,
                "nullable-error",
                &request_body(Endpoint::Responses, "route", "hello", 8, false),
            )
            .await;
            server.abort();
            if should_pass {
                assert!(result.is_ok(), "{result:?}");
            } else {
                assert_eq!(result.unwrap_err().kind, "invalid_response");
            }
        }
    }

    #[test]
    fn request_bodies_match_public_endpoints() {
        let chat = request_body(Endpoint::Chat, "route", "hello", 8, true);
        let messages = request_body(Endpoint::Messages, "route", "hello", 8, true);
        let responses = request_body(Endpoint::Responses, "route", "hello", 8, true);

        assert_eq!(chat["messages"][0]["content"], json!("hello"));
        assert_eq!(chat["max_tokens"], json!(8));
        assert_eq!(messages["max_tokens"], json!(8));
        assert_eq!(responses["input"], json!("hello"));
        assert_eq!(responses["max_output_tokens"], json!(8));
        for body in [&chat, &messages, &responses] {
            assert_eq!(body["stream"], json!(true));
        }
    }

    #[test]
    fn parse_metrics_reads_only_required_counters() {
        let metrics = parse_metrics(
            "# TYPE switchyard_total_requests gauge\n\
             switchyard_total_requests 42\n\
             switchyard_total_errors{} 3\n\
             switchyard_requests_total{model=\"route\"} 10\n",
        );

        assert_eq!(metrics.requests, Some(42.0));
        assert_eq!(metrics.errors, Some(3.0));
    }

    #[test]
    fn stream_validator_accepts_each_public_terminal_event() -> Result<(), RequestError> {
        for (endpoint, stream) in [
            (
                Endpoint::Chat,
                "data: {\"choices\":[],\"error\":null}\n\ndata: [DONE]\n\n",
            ),
            (
                Endpoint::Messages,
                "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n",
            ),
            (
                Endpoint::Responses,
                "event: response.completed\ndata: {\"type\":\"response.completed\"}\n\n",
            ),
            (
                Endpoint::Responses,
                "event: response.incomplete\ndata: {\"type\":\"response.incomplete\"}\n\n",
            ),
        ] {
            let mut validator = StreamValidator::new(endpoint);
            for line in stream.split('\n') {
                validator.read_line(line.as_bytes())?;
            }
            validator.finish()?;
        }
        Ok(())
    }

    #[test]
    fn stream_validator_rejects_errors_and_missing_terminal_events() {
        let mut error = StreamValidator::new(Endpoint::Chat);
        let result = error.read_line(b"data: {\"error\":{\"message\":\"boom\"}}");
        assert_eq!(result.unwrap_err().kind, "stream_error");

        let mut named_error = StreamValidator::new(Endpoint::Chat);
        named_error.read_line(b"event: error").unwrap();
        assert_eq!(
            named_error
                .read_line(b"data: {\"error\":null}")
                .unwrap_err()
                .kind,
            "stream_error"
        );
        let mut typed_error = StreamValidator::new(Endpoint::Chat);
        assert_eq!(
            typed_error
                .read_line(b"data: {\"type\":\"error\",\"error\":null}")
                .unwrap_err()
                .kind,
            "stream_error"
        );

        let mut incomplete = StreamValidator::new(Endpoint::Responses);
        incomplete
            .read_line(b"event: response.output_text.delta")
            .expect("event name should parse");
        incomplete
            .read_line(b"data: {\"type\":\"response.output_text.delta\",\"delta\":\"OK\"}")
            .expect("data should parse");
        assert_eq!(incomplete.finish().unwrap_err().kind, "invalid_stream");
    }
}