Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat: switchyard rust server image (#181)

Bhuvan Agrawal11d3648a1542
diff --git a/benchmark/switchyard-rust-server.Dockerfile b/benchmark/switchyard-rust-server.Dockerfile
new file mode 100644
index 000000000..8c52b5523
--- /dev/null
+++ b/benchmark/switchyard-rust-server.Dockerfile
@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+ARG RUST_VERSION=1.96.1
+FROM rust:${RUST_VERSION}-bookworm AS builder
+
+WORKDIR /opt/switchyard
+COPY Cargo.toml Cargo.lock ./
+COPY crates ./crates
+
+RUN cargo build --locked --release -p switchyard-server
+
+FROM debian:bookworm-slim
+
+RUN apt-get update \
+ && apt-get install --no-install-recommends -y ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder \
+ /opt/switchyard/target/release/switchyard-server \
+ /usr/local/bin/switchyard-server
+
+ENV HOME=/tmp
+
+USER 1000:1000
+EXPOSE 4000
+
+ENTRYPOINT ["switchyard-server"]
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index dd3ae8ba3..6dcb5d2c5 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -16,7 +16,7 @@ use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant};
-use axum::extract::{rejection::JsonRejection, State};
+use axum::extract::{rejection::JsonRejection, DefaultBodyLimit, State};
use axum::http::header::CONTENT_TYPE;
use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
use axum::response::{IntoResponse, Response};
@@ -35,6 +35,9 @@ use crate::response::into_http_response;
/// Default TCP listen backlog used by the Rust server.
pub const DEFAULT_LISTEN_BACKLOG: u32 = 65_535;
+/// Maximum buffered JSON request size accepted by the LLM endpoints.
+pub const DEFAULT_MAX_REQUEST_BODY_BYTES: usize = 32 * 1024 * 1024;
+
const HEADER_SELECTED_MODEL: &str = "x-model-router-selected-model";
const HEADER_RATIONALE: &str = "x-model-router-rationale";
const MAX_ROUTING_HEADER_VALUE_LEN: usize = 512;
@@ -200,6 +203,7 @@ pub fn build_switchyard_router(state: ServerState) -> Router {
.route("/metrics", get(prometheus_metrics))
.route("/health", get(health))
.fallback(not_found)
+ .layer(DefaultBodyLimit::max(DEFAULT_MAX_REQUEST_BODY_BYTES))
.with_state(state)
}
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 95173c7dd..9b5328931 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -10,7 +10,7 @@ use std::io::Write;
use std::sync::Arc;
use axum::body::{Body, Bytes};
-use axum::extract::State;
+use axum::extract::{DefaultBodyLimit, State};
use axum::http::{Request as HttpRequest, StatusCode};
use axum::response::sse::{Event, Sse};
use axum::response::{IntoResponse, Response as HttpResponse};
@@ -47,6 +47,7 @@ impl MockUpstream {
let app = Router::new()
.route("/v1/chat/completions", post(upstream_chat))
.route("/v1/messages/count_tokens", post(upstream_count_tokens))
+ .layer(DefaultBodyLimit::disable())
.with_state(Arc::clone(&calls));
let listener = TcpListener::bind("127.0.0.1:0").await?;
let addr = listener.local_addr()?;
@@ -242,6 +243,26 @@ async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {
Ok(())
}
+#[tokio::test]
+async fn accepts_requests_larger_than_the_axum_default_body_limit() -> TestResult {
+ let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
+ let content = "x".repeat(2 * 1024 * 1024);
+
+ let response = send(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({
+ "model": ROUTE_MODEL,
+ "messages": [{"role": "user", "content": content}]
+ })),
+ )
+ .await?;
+
+ assert_eq!(response.status, StatusCode::OK);
+ Ok(())
+}
+
fn load_test_config(toml: &str) -> TestResult<ServerState> {
let mut config = tempfile::Builder::new()
.prefix("switchyard-server-config-")