Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat: TLS support in switchyard-server (#53)
Graham Kinga2a939d86341
diff --git a/Cargo.lock b/Cargo.lockindex cc34de7dd..402d83896 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -58,6 +58,15 @@ dependencies = ["windows-sys 0.61.2",]+[[package]]+name = "arc-swap"+version = "1.9.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b"+dependencies = [+ "rustversion",+]+[[package]]name = "async-stream"version = "0.3.6"@@ -97,6 +106,35 @@ version = "1.1.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"+[[package]]+name = "autocfg"+version = "1.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"++[[package]]+name = "aws-lc-rs"+version = "1.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad"+dependencies = [+ "aws-lc-sys",+ "zeroize",+]++[[package]]+name = "aws-lc-sys"+version = "0.42.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444"+dependencies = [+ "cc",+ "cmake",+ "dunce",+ "fs_extra",+ "pkg-config",+]+[[package]]name = "axum"version = "0.8.9"@@ -149,6 +187,28 @@ dependencies = ["tracing",]+[[package]]+name = "axum-server"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b1df331683d982a0b9492b38127151e6453639cd34926eb9c07d4cd8c6d22bfc"+dependencies = [+ "arc-swap",+ "bytes",+ "either",+ "fs-err",+ "http",+ "http-body",+ "hyper",+ "hyper-util",+ "pin-project-lite",+ "rustls",+ "rustls-pki-types",+ "tokio",+ "tokio-rustls",+ "tower-service",+]+[[package]]name = "base64"version = "0.22.1"@@ -180,6 +240,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98"dependencies = ["find-msvc-tools",+ "jobserver",+ "libc","shlex",]@@ -235,6 +297,15 @@ version = "1.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"+[[package]]+name = "cmake"+version = "0.1.58"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678"+dependencies = [+ "cc",+]+[[package]]name = "colorchoice"version = "1.0.5"@@ -274,6 +345,18 @@ dependencies = ["syn",]+[[package]]+name = "dunce"+version = "1.0.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"++[[package]]+name = "either"+version = "1.16.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"+[[package]]name = "equivalent"version = "1.0.2"@@ -296,6 +379,12 @@ version = "0.1.9"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"+[[package]]+name = "fnv"+version = "1.0.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"+[[package]]name = "foldhash"version = "0.1.5"@@ -311,6 +400,22 @@ dependencies = ["percent-encoding",]+[[package]]+name = "fs-err"+version = "3.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b91aa448ca50d7e79433bdf3ee8d99215430d2ec02ade5aefab2a073a1822e8a"+dependencies = [+ "autocfg",+ "tokio",+]++[[package]]+name = "fs_extra"+version = "1.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"+[[package]]name = "futures-channel"version = "0.3.32"@@ -393,11 +498,41 @@ dependencies = ["cfg-if","js-sys","libc",- "r-efi",+ "r-efi 5.3.0","wasip2","wasm-bindgen",]+[[package]]+name = "getrandom"+version = "0.4.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"+dependencies = [+ "cfg-if",+ "libc",+ "r-efi 6.0.0",+]++[[package]]+name = "h2"+version = "0.4.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"+dependencies = [+ "atomic-waker",+ "bytes",+ "fnv",+ "futures-core",+ "futures-sink",+ "http",+ "indexmap",+ "slab",+ "tokio",+ "tokio-util",+ "tracing",+]+[[package]]name = "hashbrown"version = "0.15.5"@@ -476,6 +611,7 @@ dependencies = ["bytes","futures-channel","futures-core",+ "h2","http","http-body","httparse",@@ -657,6 +793,16 @@ version = "1.0.18"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"+[[package]]+name = "jobserver"+version = "0.1.35"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3"+dependencies = [+ "getrandom 0.4.3",+ "libc",+]+[[package]]name = "js-sys"version = "0.3.98"@@ -799,6 +945,12 @@ version = "0.2.17"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"+[[package]]+name = "pkg-config"+version = "0.3.33"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"+[[package]]name = "portable-atomic"version = "1.13.1"@@ -994,6 +1146,12 @@ version = "5.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"+[[package]]+name = "r-efi"+version = "6.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"+[[package]]name = "rand"version = "0.8.6"@@ -1129,6 +1287,8 @@ version = "0.23.40"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"dependencies = [+ "aws-lc-rs",+ "log","once_cell","ring","rustls-pki-types",@@ -1165,6 +1325,7 @@ version = "0.103.13"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"dependencies = [+ "aws-lc-rs","ring","rustls-pki-types","untrusted",@@ -1440,9 +1601,11 @@ dependencies = ["async-stream","async-trait","axum",+ "axum-server","clap","futures-util","http-body-util",+ "rustls","serde","serde_json","switchyard-components-v2",diff --git a/crates/switchyard-py/src/server_bindings.rs b/crates/switchyard-py/src/server_bindings.rsindex bab8435f8..a1b5cec87 100644--- a/crates/switchyard-py/src/server_bindings.rs+++ b/crates/switchyard-py/src/server_bindings.rs@@ -39,6 +39,7 @@ fn run_profile_server(addr: SocketAddr::new(ip, port),backlog,dry_run,+ tls: None,};// `detach` runs synchronously with the GIL released, so startup errors stilldiff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.tomlindex 6988a1fb2..23012b853 100644--- a/crates/switchyard-server/Cargo.toml+++ b/crates/switchyard-server/Cargo.toml@@ -14,6 +14,8 @@ rust-version.workspace = true[dependencies]async-stream = "0.3"axum = "0.8"+axum-server = { version = "0.8", features = ["tls-rustls"] }+rustls = { version = "0.23" }clap = { version = "4", features = ["derive", "env"] }futures-util = "0.3"serde = { version = "1", features = ["derive"] }diff --git a/crates/switchyard-server/src/cli.rs b/crates/switchyard-server/src/cli.rsindex 1056acb21..d71008ce6 100644--- a/crates/switchyard-server/src/cli.rs+++ b/crates/switchyard-server/src/cli.rs@@ -7,8 +7,8 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr};use std::path::PathBuf;use clap::Parser;-use switchyard_core::Result;-use switchyard_server::{run_server, ServerRunOptions, DEFAULT_LISTEN_BACKLOG};+use switchyard_core::{Result, SwitchyardError};+use switchyard_server::{run_server, ServerRunOptions, TLSOptions, DEFAULT_LISTEN_BACKLOG};const DEFAULT_HOST: IpAddr = IpAddr::V4(Ipv4Addr::UNSPECIFIED);const DEFAULT_PORT: u16 = 4000;@@ -40,6 +40,14 @@ pub(crate) struct ServerArgs {/// Validate and build the config without starting the HTTP listener.#[arg(long)]pub(crate) dry_run: bool,++ /// TLS certificate path, PEM format+ #[arg(long, requires = "tls_key")]+ pub(crate) tls_cert: Option<PathBuf>,++ /// TLS certificate key path, PEM format+ #[arg(long, requires = "tls_cert")]+ pub(crate) tls_key: Option<PathBuf>,}impl ServerArgs {@@ -48,17 +56,30 @@ impl ServerArgs {Self::parse()}- fn into_options(self) -> ServerRunOptions {- ServerRunOptions {+ fn into_options(self) -> Result<ServerRunOptions> {+ let mut tls_options = None;+ if let (Some(cert), Some(key)) = (self.tls_cert, self.tls_key) {+ if !cert.exists() || !key.exists() {+ return Err(SwitchyardError::InvalidConfig(format!(+ "Invalid path in --tls-cert {} or --tls-key {}. File does not exist.",+ cert.display(),+ key.display()+ )));+ }+ tls_options = Some(TLSOptions { cert, key })+ };+ Ok(ServerRunOptions {config: self.config,addr: SocketAddr::new(self.host, self.port),backlog: self.backlog,dry_run: self.dry_run,- }+ tls: tls_options,+ })}}/// Loads config, optionally validates it, then starts the Rust server.pub(crate) async fn run(args: ServerArgs) -> Result<()> {- run_server(args.into_options()).await+ let opts = args.into_options()?;+ run_server(opts).await}diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex d5d04805f..855efeb76 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -15,20 +15,23 @@ use std::collections::BTreeMap;use std::net::SocketAddr;use std::path::{Path, PathBuf};use std::sync::Arc;+use std::time::Duration;use axum::extract::{rejection::JsonRejection, State};use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode};use axum::response::{IntoResponse, Response};use axum::routing::{get, post};use axum::{Json, Router};+use axum_server::tls_rustls::RustlsConfig;use serde_json::{json, Value};+use tokio::net::{TcpListener, TcpSocket};+use switchyard_components_v2::{parse_profile_config_path, profile_stats_accumulator, ProfileConfigPlan, ProfileInput,ProfileResponse, RequestMetadata, RoutingMetadata,};use switchyard_core::{ChatRequest, ChatRequestType, RequestId, Result, SwitchyardError};use switchyard_translation::{TranslationEngine, TranslationPolicy, WireFormat};-use tokio::net::{TcpListener, TcpSocket};pub use registry::{ProfileRegistry, ServedModel};@@ -91,6 +94,24 @@ pub struct ServerRunOptions {pub backlog: u32,/// Validate and print public model IDs without binding a socket.pub dry_run: bool,+ /// If this is set server is HTTPS.+ /// Caller must validate that the files exist.+ pub tls: Option<TLSOptions>,+}++#[derive(Clone, Debug)]+pub struct TLSOptions {+ /// TLS certificate path, PEM format+ pub cert: PathBuf,++ /// TLS certificate key path, PEM format+ pub key: PathBuf,+}++impl ServerRunOptions {+ fn is_tls(&self) -> bool {+ self.tls.is_some()+ }}/// Builds a server state by loading and resolving a profile config path.@@ -111,12 +132,53 @@ pub async fn run_server(options: ServerRunOptions) -> Result<()> {let listener = bind_tcp_listener(options.addr, options.backlog)?;let bound_addr = listener.local_addr().map_err(server_io_error)?;- let banner_options = ServerRunOptions {+ let server_options = ServerRunOptions {addr: bound_addr,..options};- eprintln!("{}", startup_banner(&banner_options, state.registry()));- serve(listener, state).await+ eprintln!("{}", startup_banner(&server_options, state.registry()));+ let router = build_switchyard_router(state);+ if let Some(tls) = server_options.tls {+ serve_tls(listener, router, tls).await+ } else {+ serve(listener, router).await+ }+}++async fn serve_tls(listener: TcpListener, router: Router, tls: TLSOptions) -> Result<()> {+ // aws_lc_rs is the default but other crates pull in `ring` also,+ // so rustls doesn't know which one to use. Tell it.+ if let Err(e) = rustls::crypto::aws_lc_rs::default_provider().install_default() {+ eprintln!("TLS crypto provider already installed: {e:?}");+ }++ let config = RustlsConfig::from_pem_file(tls.cert, tls.key)+ .await+ .map_err(server_io_error)?;+ let handle = axum_server::Handle::new();++ let shutdown_handle = handle.clone();+ tokio::spawn(async move {+ shutdown_signal().await;+ // Drain connections. Later should loop on connection_count() > 0.+ shutdown_handle.graceful_shutdown(Some(Duration::from_secs(2)));+ });++ let std_listener = listener.into_std().map_err(server_io_error)?;+ axum_server::from_tcp_rustls(std_listener, config)+ .map_err(server_io_error)?+ .handle(handle.clone())+ .serve(router.into_make_service())+ .await+ .map_err(server_io_error)+}++/// Serves a Switchyard router on an already-bound TCP listener.+async fn serve(listener: TcpListener, router: Router) -> Result<()> {+ axum::serve(listener, router)+ .with_graceful_shutdown(shutdown_signal())+ .await+ .map_err(server_io_error)}/// Builds an Axum router with the same primary endpoint paths as the Python app.@@ -137,14 +199,6 @@ pub fn build_switchyard_router(state: ServerState) -> Router {.with_state(state)}-/// Serves a Switchyard router on an already-bound TCP listener.-async fn serve(listener: TcpListener, state: ServerState) -> Result<()> {- axum::serve(listener, build_switchyard_router(state))- .with_graceful_shutdown(shutdown_signal())- .await- .map_err(server_io_error)-}-fn bind_tcp_listener(addr: SocketAddr, backlog: u32) -> Result<TcpListener> {let socket = if addr.is_ipv4() {TcpSocket::new_v4()@@ -500,8 +554,9 @@ fn model_entry_json(entry: &ServedModel) -> Value {fn startup_banner(options: &ServerRunOptions, registry: &ProfileRegistry) -> String {let entries = registry.served_models();- let listen_url = url_for_addr(options.addr);- let local_url = local_url_for_addr(options.addr);+ let scheme = if options.is_tls() { "https" } else { "http" };+ let listen_url = url_for_addr(scheme, options.addr);+ let local_url = local_url_for_addr(scheme, options.addr);let mut output = String::new();push_line(&mut output, "Switchyard Rust profile server");@@ -542,7 +597,7 @@ fn startup_banner(options: &ServerRunOptions, registry: &ProfileRegistry) -> Strlet payload = json!({"model": entry.id.as_str(),"messages": [{"role": "user", "content": "Say ok"}],- "max_tokens": 8,+ "max_tokens": 256, // Space for thinking tokens});push_line(&mut output,@@ -552,6 +607,12 @@ fn startup_banner(options: &ServerRunOptions, registry: &ProfileRegistry) -> Str),);}+ if options.is_tls() {+ push_line(+ &mut output,+ " For self-signed certificates use `curl --insecure ..`",+ );+ }push_line(&mut output, "");push_line(&mut output, " stop: Ctrl-C");output@@ -579,17 +640,17 @@ fn push_line(output: &mut String, line: impl AsRef<str>) {output.push('\n');}-fn url_for_addr(addr: SocketAddr) -> String {- format!("http://{}:{}", host_for_url(addr.ip()), addr.port())+fn url_for_addr(scheme: &'static str, addr: SocketAddr) -> String {+ format!("{scheme}://{}:{}", host_for_url(addr.ip()), addr.port())}-fn local_url_for_addr(addr: SocketAddr) -> String {+fn local_url_for_addr(scheme: &'static str, addr: SocketAddr) -> String {let host = match addr.ip() {std::net::IpAddr::V4(ip) if ip.is_unspecified() => "127.0.0.1".to_string(),std::net::IpAddr::V6(ip) if ip.is_unspecified() => "[::1]".to_string(),ip => host_for_url(ip),};- format!("http://{host}:{}", addr.port())+ format!("{scheme}://{host}:{}", addr.port())}fn host_for_url(ip: std::net::IpAddr) -> String {