Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat: TLS support in switchyard-server (#53)

Graham Kinga2a939d86341
diff --git a/Cargo.lock b/Cargo.lock
index cc34de7dd..402d83896 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -58,6 +58,15 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "arc-swap"
+version = "1.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b"
+dependencies = [
+ "rustversion",
+]
+
[[package]]
name = "async-stream"
version = "0.3.6"
@@ -97,6 +106,35 @@ version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
+[[package]]
+name = "aws-lc-rs"
+version = "1.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad"
+dependencies = [
+ "aws-lc-sys",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-lc-sys"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444"
+dependencies = [
+ "cc",
+ "cmake",
+ "dunce",
+ "fs_extra",
+ "pkg-config",
+]
+
[[package]]
name = "axum"
version = "0.8.9"
@@ -149,6 +187,28 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "axum-server"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1df331683d982a0b9492b38127151e6453639cd34926eb9c07d4cd8c6d22bfc"
+dependencies = [
+ "arc-swap",
+ "bytes",
+ "either",
+ "fs-err",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-util",
+ "pin-project-lite",
+ "rustls",
+ "rustls-pki-types",
+ "tokio",
+ "tokio-rustls",
+ "tower-service",
+]
+
[[package]]
name = "base64"
version = "0.22.1"
@@ -180,6 +240,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98"
dependencies = [
"find-msvc-tools",
+ "jobserver",
+ "libc",
"shlex",
]
@@ -235,6 +297,15 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
+[[package]]
+name = "cmake"
+version = "0.1.58"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678"
+dependencies = [
+ "cc",
+]
+
[[package]]
name = "colorchoice"
version = "1.0.5"
@@ -274,6 +345,18 @@ dependencies = [
"syn",
]
+[[package]]
+name = "dunce"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
+
+[[package]]
+name = "either"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
+
[[package]]
name = "equivalent"
version = "1.0.2"
@@ -296,6 +379,12 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
[[package]]
name = "foldhash"
version = "0.1.5"
@@ -311,6 +400,22 @@ dependencies = [
"percent-encoding",
]
+[[package]]
+name = "fs-err"
+version = "3.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b91aa448ca50d7e79433bdf3ee8d99215430d2ec02ade5aefab2a073a1822e8a"
+dependencies = [
+ "autocfg",
+ "tokio",
+]
+
+[[package]]
+name = "fs_extra"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
+
[[package]]
name = "futures-channel"
version = "0.3.32"
@@ -393,11 +498,41 @@ dependencies = [
"cfg-if",
"js-sys",
"libc",
- "r-efi",
+ "r-efi 5.3.0",
"wasip2",
"wasm-bindgen",
]
+[[package]]
+name = "getrandom"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi 6.0.0",
+]
+
+[[package]]
+name = "h2"
+version = "0.4.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"
+dependencies = [
+ "atomic-waker",
+ "bytes",
+ "fnv",
+ "futures-core",
+ "futures-sink",
+ "http",
+ "indexmap",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tracing",
+]
+
[[package]]
name = "hashbrown"
version = "0.15.5"
@@ -476,6 +611,7 @@ dependencies = [
"bytes",
"futures-channel",
"futures-core",
+ "h2",
"http",
"http-body",
"httparse",
@@ -657,6 +793,16 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+[[package]]
+name = "jobserver"
+version = "0.1.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3"
+dependencies = [
+ "getrandom 0.4.3",
+ "libc",
+]
+
[[package]]
name = "js-sys"
version = "0.3.98"
@@ -799,6 +945,12 @@ version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
+[[package]]
+name = "pkg-config"
+version = "0.3.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
+
[[package]]
name = "portable-atomic"
version = "1.13.1"
@@ -994,6 +1146,12 @@ version = "5.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
+[[package]]
+name = "r-efi"
+version = "6.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
+
[[package]]
name = "rand"
version = "0.8.6"
@@ -1129,6 +1287,8 @@ version = "0.23.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"
dependencies = [
+ "aws-lc-rs",
+ "log",
"once_cell",
"ring",
"rustls-pki-types",
@@ -1165,6 +1325,7 @@ version = "0.103.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
dependencies = [
+ "aws-lc-rs",
"ring",
"rustls-pki-types",
"untrusted",
@@ -1440,9 +1601,11 @@ dependencies = [
"async-stream",
"async-trait",
"axum",
+ "axum-server",
"clap",
"futures-util",
"http-body-util",
+ "rustls",
"serde",
"serde_json",
"switchyard-components-v2",
diff --git a/crates/switchyard-py/src/server_bindings.rs b/crates/switchyard-py/src/server_bindings.rs
index bab8435f8..a1b5cec87 100644
--- a/crates/switchyard-py/src/server_bindings.rs
+++ b/crates/switchyard-py/src/server_bindings.rs
@@ -39,6 +39,7 @@ fn run_profile_server(
addr: SocketAddr::new(ip, port),
backlog,
dry_run,
+ tls: None,
};
// `detach` runs synchronously with the GIL released, so startup errors still
diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.toml
index 6988a1fb2..23012b853 100644
--- a/crates/switchyard-server/Cargo.toml
+++ b/crates/switchyard-server/Cargo.toml
@@ -14,6 +14,8 @@ rust-version.workspace = true
[dependencies]
async-stream = "0.3"
axum = "0.8"
+axum-server = { version = "0.8", features = ["tls-rustls"] }
+rustls = { version = "0.23" }
clap = { version = "4", features = ["derive", "env"] }
futures-util = "0.3"
serde = { version = "1", features = ["derive"] }
diff --git a/crates/switchyard-server/src/cli.rs b/crates/switchyard-server/src/cli.rs
index 1056acb21..d71008ce6 100644
--- a/crates/switchyard-server/src/cli.rs
+++ b/crates/switchyard-server/src/cli.rs
@@ -7,8 +7,8 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::path::PathBuf;
use clap::Parser;
-use switchyard_core::Result;
-use switchyard_server::{run_server, ServerRunOptions, DEFAULT_LISTEN_BACKLOG};
+use switchyard_core::{Result, SwitchyardError};
+use switchyard_server::{run_server, ServerRunOptions, TLSOptions, DEFAULT_LISTEN_BACKLOG};
const DEFAULT_HOST: IpAddr = IpAddr::V4(Ipv4Addr::UNSPECIFIED);
const DEFAULT_PORT: u16 = 4000;
@@ -40,6 +40,14 @@ pub(crate) struct ServerArgs {
/// Validate and build the config without starting the HTTP listener.
#[arg(long)]
pub(crate) dry_run: bool,
+
+ /// TLS certificate path, PEM format
+ #[arg(long, requires = "tls_key")]
+ pub(crate) tls_cert: Option<PathBuf>,
+
+ /// TLS certificate key path, PEM format
+ #[arg(long, requires = "tls_cert")]
+ pub(crate) tls_key: Option<PathBuf>,
}
impl ServerArgs {
@@ -48,17 +56,30 @@ impl ServerArgs {
Self::parse()
}
- fn into_options(self) -> ServerRunOptions {
- ServerRunOptions {
+ fn into_options(self) -> Result<ServerRunOptions> {
+ let mut tls_options = None;
+ if let (Some(cert), Some(key)) = (self.tls_cert, self.tls_key) {
+ if !cert.exists() || !key.exists() {
+ return Err(SwitchyardError::InvalidConfig(format!(
+ "Invalid path in --tls-cert {} or --tls-key {}. File does not exist.",
+ cert.display(),
+ key.display()
+ )));
+ }
+ tls_options = Some(TLSOptions { cert, key })
+ };
+ Ok(ServerRunOptions {
config: self.config,
addr: SocketAddr::new(self.host, self.port),
backlog: self.backlog,
dry_run: self.dry_run,
- }
+ tls: tls_options,
+ })
}
}
/// Loads config, optionally validates it, then starts the Rust server.
pub(crate) async fn run(args: ServerArgs) -> Result<()> {
- run_server(args.into_options()).await
+ let opts = args.into_options()?;
+ run_server(opts).await
}
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index d5d04805f..855efeb76 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -15,20 +15,23 @@ use std::collections::BTreeMap;
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use std::sync::Arc;
+use std::time::Duration;
use axum::extract::{rejection::JsonRejection, State};
use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
use axum::{Json, Router};
+use axum_server::tls_rustls::RustlsConfig;
use serde_json::{json, Value};
+use tokio::net::{TcpListener, TcpSocket};
+
use switchyard_components_v2::{
parse_profile_config_path, profile_stats_accumulator, ProfileConfigPlan, ProfileInput,
ProfileResponse, RequestMetadata, RoutingMetadata,
};
use switchyard_core::{ChatRequest, ChatRequestType, RequestId, Result, SwitchyardError};
use switchyard_translation::{TranslationEngine, TranslationPolicy, WireFormat};
-use tokio::net::{TcpListener, TcpSocket};
pub use registry::{ProfileRegistry, ServedModel};
@@ -91,6 +94,24 @@ pub struct ServerRunOptions {
pub backlog: u32,
/// Validate and print public model IDs without binding a socket.
pub dry_run: bool,
+ /// If this is set server is HTTPS.
+ /// Caller must validate that the files exist.
+ pub tls: Option<TLSOptions>,
+}
+
+#[derive(Clone, Debug)]
+pub struct TLSOptions {
+ /// TLS certificate path, PEM format
+ pub cert: PathBuf,
+
+ /// TLS certificate key path, PEM format
+ pub key: PathBuf,
+}
+
+impl ServerRunOptions {
+ fn is_tls(&self) -> bool {
+ self.tls.is_some()
+ }
}
/// Builds a server state by loading and resolving a profile config path.
@@ -111,12 +132,53 @@ pub async fn run_server(options: ServerRunOptions) -> Result<()> {
let listener = bind_tcp_listener(options.addr, options.backlog)?;
let bound_addr = listener.local_addr().map_err(server_io_error)?;
- let banner_options = ServerRunOptions {
+ let server_options = ServerRunOptions {
addr: bound_addr,
..options
};
- eprintln!("{}", startup_banner(&banner_options, state.registry()));
- serve(listener, state).await
+ eprintln!("{}", startup_banner(&server_options, state.registry()));
+ let router = build_switchyard_router(state);
+ if let Some(tls) = server_options.tls {
+ serve_tls(listener, router, tls).await
+ } else {
+ serve(listener, router).await
+ }
+}
+
+async fn serve_tls(listener: TcpListener, router: Router, tls: TLSOptions) -> Result<()> {
+ // aws_lc_rs is the default but other crates pull in `ring` also,
+ // so rustls doesn't know which one to use. Tell it.
+ if let Err(e) = rustls::crypto::aws_lc_rs::default_provider().install_default() {
+ eprintln!("TLS crypto provider already installed: {e:?}");
+ }
+
+ let config = RustlsConfig::from_pem_file(tls.cert, tls.key)
+ .await
+ .map_err(server_io_error)?;
+ let handle = axum_server::Handle::new();
+
+ let shutdown_handle = handle.clone();
+ tokio::spawn(async move {
+ shutdown_signal().await;
+ // Drain connections. Later should loop on connection_count() > 0.
+ shutdown_handle.graceful_shutdown(Some(Duration::from_secs(2)));
+ });
+
+ let std_listener = listener.into_std().map_err(server_io_error)?;
+ axum_server::from_tcp_rustls(std_listener, config)
+ .map_err(server_io_error)?
+ .handle(handle.clone())
+ .serve(router.into_make_service())
+ .await
+ .map_err(server_io_error)
+}
+
+/// Serves a Switchyard router on an already-bound TCP listener.
+async fn serve(listener: TcpListener, router: Router) -> Result<()> {
+ axum::serve(listener, router)
+ .with_graceful_shutdown(shutdown_signal())
+ .await
+ .map_err(server_io_error)
}
/// Builds an Axum router with the same primary endpoint paths as the Python app.
@@ -137,14 +199,6 @@ pub fn build_switchyard_router(state: ServerState) -> Router {
.with_state(state)
}
-/// Serves a Switchyard router on an already-bound TCP listener.
-async fn serve(listener: TcpListener, state: ServerState) -> Result<()> {
- axum::serve(listener, build_switchyard_router(state))
- .with_graceful_shutdown(shutdown_signal())
- .await
- .map_err(server_io_error)
-}
-
fn bind_tcp_listener(addr: SocketAddr, backlog: u32) -> Result<TcpListener> {
let socket = if addr.is_ipv4() {
TcpSocket::new_v4()
@@ -500,8 +554,9 @@ fn model_entry_json(entry: &ServedModel) -> Value {
fn startup_banner(options: &ServerRunOptions, registry: &ProfileRegistry) -> String {
let entries = registry.served_models();
- let listen_url = url_for_addr(options.addr);
- let local_url = local_url_for_addr(options.addr);
+ let scheme = if options.is_tls() { "https" } else { "http" };
+ let listen_url = url_for_addr(scheme, options.addr);
+ let local_url = local_url_for_addr(scheme, options.addr);
let mut output = String::new();
push_line(&mut output, "Switchyard Rust profile server");
@@ -542,7 +597,7 @@ fn startup_banner(options: &ServerRunOptions, registry: &ProfileRegistry) -> Str
let payload = json!({
"model": entry.id.as_str(),
"messages": [{"role": "user", "content": "Say ok"}],
- "max_tokens": 8,
+ "max_tokens": 256, // Space for thinking tokens
});
push_line(
&mut output,
@@ -552,6 +607,12 @@ fn startup_banner(options: &ServerRunOptions, registry: &ProfileRegistry) -> Str
),
);
}
+ if options.is_tls() {
+ push_line(
+ &mut output,
+ " For self-signed certificates use `curl --insecure ..`",
+ );
+ }
push_line(&mut output, "");
push_line(&mut output, " stop: Ctrl-C");
output
@@ -579,17 +640,17 @@ fn push_line(output: &mut String, line: impl AsRef<str>) {
output.push('\n');
}
-fn url_for_addr(addr: SocketAddr) -> String {
- format!("http://{}:{}", host_for_url(addr.ip()), addr.port())
+fn url_for_addr(scheme: &'static str, addr: SocketAddr) -> String {
+ format!("{scheme}://{}:{}", host_for_url(addr.ip()), addr.port())
}
-fn local_url_for_addr(addr: SocketAddr) -> String {
+fn local_url_for_addr(scheme: &'static str, addr: SocketAddr) -> String {
let host = match addr.ip() {
std::net::IpAddr::V4(ip) if ip.is_unspecified() => "127.0.0.1".to_string(),
std::net::IpAddr::V6(ip) if ip.is_unspecified() => "[::1]".to_string(),
ip => host_for_url(ip),
};
- format!("http://{host}:{}", addr.port())
+ format!("{scheme}://{host}:{}", addr.port())
}
fn host_for_url(ip: std::net::IpAddr) -> String {