Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(server): add native Python host with OpenTelemetry (#210)

nachiketb-nvidia50e8d74dbead
diff --git a/Cargo.lock b/Cargo.lock
index 5cf8b253d..ab0d4052a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -61,6 +61,12 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "anyhow"
+version = "1.0.104"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
+
[[package]]
name = "arc-swap"
version = "1.9.2"
@@ -336,6 +342,32 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
+[[package]]
+name = "combine"
+version = "4.6.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd"
+dependencies = [
+ "bytes",
+ "memchr",
+]
+
+[[package]]
+name = "core-foundation"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
+
[[package]]
name = "cpufeatures"
version = "0.3.0"
@@ -846,12 +878,70 @@ version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
+[[package]]
+name = "itertools"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
+dependencies = [
+ "either",
+]
+
[[package]]
name = "itoa"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+[[package]]
+name = "jni"
+version = "0.22.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498"
+dependencies = [
+ "cfg-if",
+ "combine",
+ "jni-macros",
+ "jni-sys",
+ "log",
+ "simd_cesu8",
+ "thiserror 2.0.18",
+ "walkdir",
+ "windows-link",
+]
+
+[[package]]
+name = "jni-macros"
+version = "0.22.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "rustc_version",
+ "simd_cesu8",
+ "syn",
+]
+
+[[package]]
+name = "jni-sys"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"
+dependencies = [
+ "jni-sys-macros",
+]
+
+[[package]]
+name = "jni-sys-macros"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
+dependencies = [
+ "quote",
+ "syn",
+]
+
[[package]]
name = "jobserver"
version = "0.1.35"
@@ -987,6 +1077,12 @@ version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
+[[package]]
+name = "openssl-probe"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
+
[[package]]
name = "opentelemetry"
version = "0.32.0"
@@ -1001,6 +1097,35 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "opentelemetry-http"
+version = "0.32.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "http",
+ "opentelemetry",
+ "reqwest 0.13.4",
+]
+
+[[package]]
+name = "opentelemetry-otlp"
+version = "0.32.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35"
+dependencies = [
+ "http",
+ "opentelemetry",
+ "opentelemetry-http",
+ "opentelemetry-proto",
+ "opentelemetry_sdk",
+ "prost",
+ "reqwest 0.13.4",
+ "thiserror 2.0.18",
+]
+
[[package]]
name = "opentelemetry-prometheus"
version = "0.32.0"
@@ -1014,6 +1139,17 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "opentelemetry-proto"
+version = "0.32.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638"
+dependencies = [
+ "opentelemetry",
+ "opentelemetry_sdk",
+ "prost",
+]
+
[[package]]
name = "opentelemetry_sdk"
version = "0.32.1"
@@ -1130,6 +1266,29 @@ dependencies = [
"thiserror 2.0.18",
]
+[[package]]
+name = "prost"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1"
+dependencies = [
+ "bytes",
+ "prost-derive",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
+dependencies = [
+ "anyhow",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "protobuf"
version = "3.7.2"
@@ -1258,6 +1417,7 @@ version = "0.11.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560"
dependencies = [
+ "aws-lc-rs",
"bytes",
"getrandom 0.4.3",
"lru-slab",
@@ -1473,6 +1633,43 @@ dependencies = [
"webpki-roots",
]
+[[package]]
+name = "reqwest"
+version = "0.13.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
+dependencies = [
+ "base64",
+ "bytes",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "http",
+ "http-body",
+ "http-body-util",
+ "hyper",
+ "hyper-rustls",
+ "hyper-util",
+ "js-sys",
+ "log",
+ "percent-encoding",
+ "pin-project-lite",
+ "quinn",
+ "rustls",
+ "rustls-pki-types",
+ "rustls-platform-verifier",
+ "sync_wrapper",
+ "tokio",
+ "tokio-rustls",
+ "tower",
+ "tower-http",
+ "tower-service",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
[[package]]
name = "ring"
version = "0.17.14"
@@ -1493,6 +1690,15 @@ version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
+[[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+dependencies = [
+ "semver",
+]
+
[[package]]
name = "rustix"
version = "1.1.4"
@@ -1521,6 +1727,18 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "rustls-native-certs"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d"
+dependencies = [
+ "openssl-probe",
+ "rustls-pki-types",
+ "schannel",
+ "security-framework",
+]
+
[[package]]
name = "rustls-pki-types"
version = "1.15.0"
@@ -1531,6 +1749,33 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "rustls-platform-verifier"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0"
+dependencies = [
+ "core-foundation",
+ "core-foundation-sys",
+ "jni",
+ "log",
+ "once_cell",
+ "rustls",
+ "rustls-native-certs",
+ "rustls-platform-verifier-android",
+ "rustls-webpki",
+ "security-framework",
+ "security-framework-sys",
+ "webpki-root-certs",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "rustls-platform-verifier-android"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
+
[[package]]
name = "rustls-webpki"
version = "0.103.13"
@@ -1555,12 +1800,59 @@ version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "schannel"
+version = "0.1.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
+dependencies = [
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+[[package]]
+name = "security-framework"
+version = "3.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
+dependencies = [
+ "bitflags",
+ "core-foundation",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "semver"
+version = "1.0.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
+
[[package]]
name = "serde"
version = "1.0.228"
@@ -1661,6 +1953,22 @@ dependencies = [
"libc",
]
+[[package]]
+name = "simd_cesu8"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520"
+dependencies = [
+ "rustc_version",
+ "simdutf8",
+]
+
+[[package]]
+name = "simdutf8"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
+
[[package]]
name = "slab"
version = "0.4.12"
@@ -1711,7 +2019,7 @@ dependencies = [
"futures-util",
"parking_lot",
"rand 0.8.7",
- "reqwest",
+ "reqwest 0.12.28",
"serde",
"serde_json",
"switchyard-libsy",
@@ -1752,7 +2060,7 @@ dependencies = [
"futures-util",
"httpdate",
"opentelemetry",
- "reqwest",
+ "reqwest 0.12.28",
"serde_json",
"switchyard-protocol",
"switchyard-translation",
@@ -1787,6 +2095,7 @@ dependencies = [
"switchyard-components",
"switchyard-libsy",
"switchyard-protocol",
+ "switchyard-server",
"switchyard-translation",
"tokio",
"tracing",
@@ -1804,6 +2113,7 @@ dependencies = [
"futures-util",
"http-body-util",
"opentelemetry",
+ "opentelemetry-otlp",
"opentelemetry-prometheus",
"opentelemetry_sdk",
"parking_lot",
@@ -1819,6 +2129,7 @@ dependencies = [
"toml",
"tower",
"tracing",
+ "tracing-opentelemetry",
"tracing-subscriber",
]
@@ -2161,6 +2472,22 @@ dependencies = [
"tracing-core",
]
+[[package]]
+name = "tracing-opentelemetry"
+version = "0.33.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26"
+dependencies = [
+ "js-sys",
+ "opentelemetry",
+ "smallvec",
+ "tracing",
+ "tracing-core",
+ "tracing-log",
+ "tracing-subscriber",
+ "web-time",
+]
+
[[package]]
name = "tracing-subscriber"
version = "0.3.23"
@@ -2227,6 +2554,16 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
+[[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
[[package]]
name = "want"
version = "0.3.1"
@@ -2339,6 +2676,15 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "webpki-root-certs"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"
+dependencies = [
+ "rustls-pki-types",
+]
+
[[package]]
name = "webpki-roots"
version = "1.0.9"
@@ -2348,6 +2694,15 @@ dependencies = [
"rustls-pki-types",
]
+[[package]]
+name = "winapi-util"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
+dependencies = [
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "windows-link"
version = "0.2.1"
diff --git a/Cargo.toml b/Cargo.toml
index 7fd0c69cf..8bf001d3d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -36,6 +36,7 @@ switchyard-components = { path = "crates/switchyard-components", version = "0.1.
switchyard-libsy = { path = "crates/libsy", version = "0.1.0" }
switchyard-llm-client = { path = "crates/libsy-llm-client", version = "0.1.0" }
switchyard-protocol = { path = "crates/protocol", version = "0.1.0" }
+switchyard-server = { path = "crates/switchyard-server", version = "0.1.0" }
switchyard-translation = { path = "crates/switchyard-translation", version = "0.1.0" }
thiserror = "2"
tokio = { version = "1", features = ["full"] }
diff --git a/crates/switchyard-py/Cargo.toml b/crates/switchyard-py/Cargo.toml
index 90d2e76f5..56dd7e3c2 100644
--- a/crates/switchyard-py/Cargo.toml
+++ b/crates/switchyard-py/Cargo.toml
@@ -26,6 +26,7 @@ pythonize = "0.28.0"
serde.workspace = true
serde_json.workspace = true
switchyard-protocol.workspace = true
+switchyard-server.workspace = true
switchyard-components.workspace = true
switchyard-translation.workspace = true
tokio.workspace = true
diff --git a/crates/switchyard-py/src/lib.rs b/crates/switchyard-py/src/lib.rs
index f9862c69f..3082697a1 100644
--- a/crates/switchyard-py/src/lib.rs
+++ b/crates/switchyard-py/src/lib.rs
@@ -8,6 +8,7 @@ mod errors;
mod interop;
mod libsy_bindings;
mod py_serde;
+mod server_bindings;
mod translation;
#[pymodule]
@@ -17,5 +18,6 @@ fn _switchyard_rust(module: &Bound<'_, PyModule>) -> PyResult<()> {
interop::register(module)?;
component_bindings::register(module)?;
libsy_bindings::register(module)?;
+ server_bindings::register(module)?;
Ok(())
}
diff --git a/crates/switchyard-py/src/server_bindings.rs b/crates/switchyard-py/src/server_bindings.rs
new file mode 100644
index 000000000..63caffe9f
--- /dev/null
+++ b/crates/switchyard-py/src/server_bindings.rs
@@ -0,0 +1,162 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Self-contained Python host for the Rust Switchyard server.
+
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+use std::path::PathBuf;
+use std::sync::mpsc::{sync_channel, Receiver, RecvTimeoutError};
+use std::time::Duration;
+
+use pyo3::exceptions::{PyRuntimeError, PyValueError};
+use pyo3::prelude::*;
+use switchyard_server::config::load_server_state;
+use switchyard_server::{
+ flush_observability, initialize_observability, BoundServer, ServerResult, ServerRunOptions,
+ DEFAULT_LISTEN_BACKLOG,
+};
+use tokio::sync::oneshot;
+use tokio::task::JoinHandle;
+
+const DEFAULT_SHUTDOWN_TIMEOUT_SECS: f64 = 2.0;
+
+/// Running loopback server backed entirely by the native Rust implementation.
+#[pyclass(name = "Server", module = "switchyard_rust.server", unsendable)]
+struct PyServer {
+ addr: SocketAddr,
+ shutdown: Option<oneshot::Sender<()>>,
+ completion: Option<Receiver<ServerResult<()>>>,
+ task: Option<JoinHandle<()>>,
+}
+
+#[pymethods]
+impl PyServer {
+ /// Loads a TOML deployment and starts serving it on loopback.
+ #[new]
+ #[pyo3(signature = (config, *, port=0))]
+ fn new(config: PathBuf, port: u16) -> PyResult<Self> {
+ initialize_observability().map_err(server_error)?;
+ let state = load_server_state(config).map_err(server_error)?;
+ let runtime = pyo3_async_runtimes::tokio::get_runtime();
+ let server = {
+ let _guard = runtime.enter();
+ BoundServer::bind(
+ state,
+ ServerRunOptions {
+ addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port),
+ backlog: DEFAULT_LISTEN_BACKLOG,
+ dry_run: false,
+ tls: None,
+ },
+ )
+ .map_err(server_error)?
+ };
+ let addr = server.local_addr();
+ let (shutdown, shutdown_receiver) = oneshot::channel();
+ let (completion_sender, completion) = sync_channel(1);
+ let task = runtime.spawn(async move {
+ let result = server
+ .serve(async move {
+ let _ = shutdown_receiver.await;
+ })
+ .await;
+ let _ = completion_sender.send(result);
+ });
+ Ok(Self {
+ addr,
+ shutdown: Some(shutdown),
+ completion: Some(completion),
+ task: Some(task),
+ })
+ }
+
+ /// Returns the bound loopback port.
+ #[getter]
+ fn port(&self) -> u16 {
+ self.addr.port()
+ }
+
+ /// Returns the HTTP base URL for clients.
+ #[getter]
+ fn base_url(&self) -> String {
+ format!("http://{}", self.addr)
+ }
+
+ /// Gracefully stops the server and flushes pending telemetry.
+ #[pyo3(signature = (*, timeout_secs=DEFAULT_SHUTDOWN_TIMEOUT_SECS))]
+ fn close(&mut self, py: Python<'_>, timeout_secs: f64) -> PyResult<()> {
+ self.close_inner(py, timeout_secs)
+ }
+
+ fn __enter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> {
+ slf
+ }
+
+ fn __exit__(
+ &mut self,
+ py: Python<'_>,
+ _exception_type: &Bound<'_, PyAny>,
+ _exception: &Bound<'_, PyAny>,
+ _traceback: &Bound<'_, PyAny>,
+ ) -> PyResult<bool> {
+ self.close_inner(py, DEFAULT_SHUTDOWN_TIMEOUT_SECS)?;
+ Ok(false)
+ }
+
+ fn __repr__(&self) -> String {
+ format!("Server(base_url={:?})", self.base_url())
+ }
+}
+
+impl PyServer {
+ fn close_inner(&mut self, py: Python<'_>, timeout_secs: f64) -> PyResult<()> {
+ let timeout = Duration::try_from_secs_f64(timeout_secs).map_err(|_| {
+ PyValueError::new_err("timeout_secs must be a supported finite non-negative number")
+ })?;
+ if let Some(shutdown) = self.shutdown.take() {
+ let _ = shutdown.send(());
+ }
+ let Some(completion) = self.completion.take() else {
+ return Ok(());
+ };
+ let task = self.task.take();
+ let result = py.detach(move || {
+ let result = completion.recv_timeout(timeout);
+ if matches!(result, Err(RecvTimeoutError::Timeout)) {
+ if let Some(task) = task {
+ task.abort();
+ }
+ }
+ flush_observability();
+ result
+ });
+ match result {
+ Ok(result) => result.map_err(server_error),
+ Err(RecvTimeoutError::Timeout) => Err(PyRuntimeError::new_err(format!(
+ "server did not stop within {timeout_secs} seconds"
+ ))),
+ Err(RecvTimeoutError::Disconnected) => Err(PyRuntimeError::new_err(
+ "server task stopped without reporting a result",
+ )),
+ }
+ }
+}
+
+impl Drop for PyServer {
+ fn drop(&mut self) {
+ if let Some(shutdown) = self.shutdown.take() {
+ let _ = shutdown.send(());
+ }
+ }
+}
+
+fn server_error(error: impl std::fmt::Display) -> PyErr {
+ PyRuntimeError::new_err(error.to_string())
+}
+
+pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
+ let server_module = PyModule::new(module.py(), "server")?;
+ server_module.add_class::<PyServer>()?;
+ module.add_submodule(&server_module)?;
+ Ok(())
+}
diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.toml
index 424d73a5f..119c311b9 100644
--- a/crates/switchyard-server/Cargo.toml
+++ b/crates/switchyard-server/Cargo.toml
@@ -20,10 +20,11 @@ rustls = { version = "0.23", default-features = false, features = ["aws-lc-rs",
clap = { version = "4", features = ["derive", "env"] }
futures-util.workspace = true
libsy = { package = "switchyard-libsy", path = "../libsy" }
-opentelemetry = { version = "0.32", default-features = false, features = ["metrics"] }
+opentelemetry = { version = "0.32", default-features = false, features = ["metrics", "trace"] }
+opentelemetry-otlp = { version = "0.32", default-features = false, features = ["http-proto", "metrics", "reqwest-blocking-client", "reqwest-rustls", "trace"] }
opentelemetry-prometheus = "0.32"
-opentelemetry_sdk = { version = "0.32", default-features = false, features = ["metrics"] }
parking_lot.workspace = true
+opentelemetry_sdk = { version = "0.32", default-features = false, features = ["metrics", "trace"] }
prometheus = "0.14"
serde.workspace = true
toml = "1.1"
@@ -32,6 +33,7 @@ serde_json.workspace = true
switchyard-translation.workspace = true
tokio.workspace = true
tracing.workspace = true
+tracing-opentelemetry = "0.33"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
[dev-dependencies]
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index db84554db..eb0a8a898 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -5,6 +5,7 @@
pub mod config;
mod metrics;
+mod observability;
mod response;
mod sse;
mod stats;
@@ -13,6 +14,7 @@ mod usage_metrics;
use std::collections::BTreeMap;
use std::error::Error;
use std::fmt::{Display, Formatter};
+use std::future::Future;
use std::net::SocketAddr;
use std::path::PathBuf;
use std::sync::Arc;
@@ -39,6 +41,8 @@ use switchyard_translation::{decode_request, WireFormat};
use crate::response::into_http_response;
use crate::stats::{prefix_probe, tracking_enabled_from_env, StatsAccumulator, StatsSnapshot};
+pub use observability::{flush_observability, initialize_observability};
+
/// Default TCP listen backlog used by the Rust server.
pub const DEFAULT_LISTEN_BACKLOG: u32 = 65_535;
@@ -156,22 +160,60 @@ pub async fn run_server(state: ServerState, options: ServerRunOptions) -> Server
return Ok(());
}
- let listener = bind_tcp_listener(options.addr, options.backlog)?;
- let bound_addr = listener.local_addr().map_err(server_io_error)?;
- let server_options = ServerRunOptions {
- addr: bound_addr,
- ..options
- };
- eprintln!("{}", startup_banner(&server_options, &state));
- let router = build_switchyard_router(state);
- if let Some(tls) = server_options.tls {
- serve_tls(listener, router, tls).await
- } else {
- serve(listener, router).await
+ let server = BoundServer::bind(state, options)?;
+ eprintln!("{}", server.startup_banner());
+ server.serve(shutdown_signal()).await
+}
+
+/// A configured server with its listening socket already bound.
+pub struct BoundServer {
+ listener: TcpListener,
+ router: Router,
+ options: ServerRunOptions,
+ state: ServerState,
+}
+
+impl BoundServer {
+ /// Binds the configured address and prepares the HTTP router.
+ pub fn bind(state: ServerState, options: ServerRunOptions) -> ServerResult<Self> {
+ let listener = bind_tcp_listener(options.addr, options.backlog)?;
+ let addr = listener.local_addr().map_err(server_io_error)?;
+ Ok(Self {
+ listener,
+ router: build_switchyard_router(state.clone()),
+ options: ServerRunOptions { addr, ..options },
+ state,
+ })
+ }
+
+ /// Returns the actual bound address, including an OS-selected port.
+ pub fn local_addr(&self) -> SocketAddr {
+ self.options.addr
+ }
+
+ /// Serves requests until the supplied shutdown future resolves.
+ pub async fn serve(
+ self,
+ shutdown: impl Future<Output = ()> + Send + 'static,
+ ) -> ServerResult<()> {
+ if let Some(tls) = self.options.tls {
+ serve_tls(self.listener, self.router, tls, shutdown).await
+ } else {
+ serve(self.listener, self.router, shutdown).await
+ }
+ }
+
+ fn startup_banner(&self) -> String {
+ startup_banner(&self.options, &self.state)
}
}
-async fn serve_tls(listener: TcpListener, router: Router, tls: TlsOptions) -> ServerResult<()> {
+async fn serve_tls(
+ listener: TcpListener,
+ router: Router,
+ tls: TlsOptions,
+ shutdown: impl Future<Output = ()> + Send + 'static,
+) -> ServerResult<()> {
if let Err(error) = rustls::crypto::aws_lc_rs::default_provider().install_default() {
tracing::debug!(?error, "TLS crypto provider was already installed");
}
@@ -183,7 +225,7 @@ async fn serve_tls(listener: TcpListener, router: Router, tls: TlsOptions) -> Se
let shutdown_handle = handle.clone();
tokio::spawn(async move {
- shutdown_signal().await;
+ shutdown.await;
shutdown_handle.graceful_shutdown(Some(Duration::from_secs(2)));
});
@@ -196,9 +238,13 @@ async fn serve_tls(listener: TcpListener, router: Router, tls: TlsOptions) -> Se
.map_err(server_io_error)
}
-async fn serve(listener: TcpListener, router: Router) -> ServerResult<()> {
+async fn serve(
+ listener: TcpListener,
+ router: Router,
+ shutdown: impl Future<Output = ()> + Send + 'static,
+) -> ServerResult<()> {
axum::serve(listener, router)
- .with_graceful_shutdown(shutdown_signal())
+ .with_graceful_shutdown(shutdown)
.await
.map_err(server_io_error)
}
diff --git a/crates/switchyard-server/src/main.rs b/crates/switchyard-server/src/main.rs
index 51c44a4be..b9f7fdf0a 100644
--- a/crates/switchyard-server/src/main.rs
+++ b/crates/switchyard-server/src/main.rs
@@ -5,36 +5,21 @@
use std::process::ExitCode;
-use tracing_subscriber::EnvFilter;
-
mod cli;
-const DEFAULT_LOG_FILTER: &str = "switchyard_server=info";
-const OPENTELEMETRY_LOG_FILTER: &str = "opentelemetry=warn";
-
#[tokio::main(flavor = "multi_thread")]
async fn main() -> ExitCode {
- if let Err(error) = init_logging() {
- eprintln!("failed to initialize logging: {error}");
+ if let Err(error) = switchyard_server::initialize_observability() {
+ eprintln!("failed to initialize observability: {error}");
return ExitCode::FAILURE;
}
- match cli::run(cli::ServerArgs::parse_args()).await {
+ let exit_code = match cli::run(cli::ServerArgs::parse_args()).await {
Ok(()) => ExitCode::SUCCESS,
Err(error) => {
eprintln!("{error}");
ExitCode::FAILURE
}
- }
-}
-
-fn init_logging() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
- let filter = EnvFilter::try_from_default_env()
- .unwrap_or_else(|_| EnvFilter::new(DEFAULT_LOG_FILTER))
- .add_directive(OPENTELEMETRY_LOG_FILTER.parse()?);
- tracing_subscriber::fmt()
- .with_env_filter(filter)
- .with_ansi(false)
- .with_writer(std::io::stderr)
- .try_init()?;
- Ok(())
+ };
+ switchyard_server::flush_observability();
+ exit_code
}
diff --git a/crates/switchyard-server/src/metrics.rs b/crates/switchyard-server/src/metrics.rs
index 8cd3df141..8be2148dc 100644
--- a/crates/switchyard-server/src/metrics.rs
+++ b/crates/switchyard-server/src/metrics.rs
@@ -40,10 +40,18 @@ fn initialize() -> Result<Metrics, String> {
.with_registry(registry.clone())
.build()
.map_err(|error| format!("failed to initialize Prometheus metrics: {error}"))?;
- let provider = SdkMeterProvider::builder()
+ let mut builder = SdkMeterProvider::builder()
.with_reader(exporter)
.with_view(routing_overhead_buckets)
- .build();
+ .with_resource(crate::observability::resource());
+ if crate::observability::otlp_enabled("METRICS") {
+ let exporter = opentelemetry_otlp::MetricExporter::builder()
+ .with_http()
+ .build()
+ .map_err(|error| format!("failed to initialize OTLP metric exporter: {error}"))?;
+ builder = builder.with_periodic_exporter(exporter);
+ }
+ let provider = builder.build();
global::set_meter_provider(provider.clone());
libsy::initialize_metrics();
global::meter("switchyard")
@@ -57,6 +65,14 @@ fn initialize() -> Result<Metrics, String> {
})
}
+pub(crate) fn flush() {
+ if let Some(Ok(metrics)) = METRICS.get() {
+ if let Err(error) = metrics._provider.force_flush() {
+ tracing::warn!(error = %error, "failed to flush OpenTelemetry metrics");
+ }
+ }
+}
+
fn routing_overhead_buckets(instrument: &Instrument) -> Option<Stream> {
if instrument.name() != "switchyard.routing_overhead_ms" {
return None;
diff --git a/crates/switchyard-server/src/observability.rs b/crates/switchyard-server/src/observability.rs
new file mode 100644
index 000000000..5c76ec3dd
--- /dev/null
+++ b/crates/switchyard-server/src/observability.rs
@@ -0,0 +1,134 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Process-wide tracing and OpenTelemetry setup for server hosts.
+
+use std::env;
+use std::sync::OnceLock;
+
+use opentelemetry::trace::TracerProvider as _;
+use opentelemetry_sdk::trace::SdkTracerProvider;
+use opentelemetry_sdk::Resource;
+use tracing_subscriber::layer::SubscriberExt as _;
+use tracing_subscriber::util::SubscriberInitExt as _;
+use tracing_subscriber::{EnvFilter, Layer as _};
+
+use crate::{metrics, ServerError, ServerResult};
+
+const DEFAULT_LOG_FILTER: &str = "switchyard_server=info";
+const OPENTELEMETRY_LOG_FILTER: &str = "opentelemetry=warn";
+const OTEL_SPAN_FILTER: &str = "libsy=info";
+const DEFAULT_SERVICE_NAME: &str = "switchyard-server";
+
+struct Observability {
+ tracer_provider: Option<SdkTracerProvider>,
+}
+
+static OBSERVABILITY: OnceLock<Result<Observability, String>> = OnceLock::new();
+
+/// Installs metrics and tracing once for either the binary or an embedded host.
+pub fn initialize_observability() -> ServerResult<()> {
+ match OBSERVABILITY.get_or_init(initialize) {
+ Ok(_) => Ok(()),
+ Err(error) => Err(ServerError::new(error.clone())),
+ }
+}
+
+/// Flushes pending OTLP telemetry without shutting down process-wide providers.
+pub fn flush_observability() {
+ if let Some(Ok(observability)) = OBSERVABILITY.get() {
+ if let Some(provider) = &observability.tracer_provider {
+ if let Err(error) = provider.force_flush() {
+ tracing::warn!(error = %error, "failed to flush OpenTelemetry traces");
+ }
+ }
+ }
+ metrics::flush();
+}
+
+pub(crate) fn otlp_enabled(signal: &str) -> bool {
+ if env_var_is_true("OTEL_SDK_DISABLED") {
+ return false;
+ }
+ if env::var(format!("OTEL_{signal}_EXPORTER"))
+ .ok()
+ .filter(|value| !value.trim().is_empty())
+ .is_some_and(|value| {
+ !value
+ .split(',')
+ .any(|exporter| exporter.trim().eq_ignore_ascii_case("otlp"))
+ })
+ {
+ return false;
+ }
+ [
+ "OTEL_EXPORTER_OTLP_ENDPOINT",
+ &format!("OTEL_EXPORTER_OTLP_{signal}_ENDPOINT"),
+ ]
+ .into_iter()
+ .any(|name| env::var(name).is_ok_and(|value| !value.trim().is_empty()))
+}
+
+pub(crate) fn resource() -> Resource {
+ let service_name = env::var("OTEL_SERVICE_NAME")
+ .ok()
+ .filter(|value| !value.trim().is_empty())
+ .unwrap_or_else(|| DEFAULT_SERVICE_NAME.to_string());
+ Resource::builder().with_service_name(service_name).build()
+}
+
+fn initialize() -> Result<Observability, String> {
+ metrics::registry()?;
+
+ let tracer_provider = otlp_enabled("TRACES")
+ .then(build_tracer_provider)
+ .transpose()?;
+ let filter = EnvFilter::try_from_default_env()
+ .unwrap_or_else(|_| EnvFilter::new(DEFAULT_LOG_FILTER))
+ .add_directive(
+ OPENTELEMETRY_LOG_FILTER
+ .parse()
+ .map_err(|error| format!("invalid OpenTelemetry log filter: {error}"))?,
+ );
+ let format = tracing_subscriber::fmt::layer()
+ .with_ansi(false)
+ .with_writer(std::io::stderr)
+ .with_filter(filter);
+
+ if let Some(provider) = &tracer_provider {
+ let tracer = provider.tracer("switchyard");
+ tracing_subscriber::registry()
+ .with(format)
+ .with(
+ tracing_opentelemetry::layer()
+ .with_tracer(tracer)
+ .with_filter(EnvFilter::new(OTEL_SPAN_FILTER)),
+ )
+ .try_init()
+ .map_err(|error| format!("failed to initialize tracing: {error}"))?;
+ } else {
+ tracing_subscriber::registry()
+ .with(format)
+ .try_init()
+ .map_err(|error| format!("failed to initialize tracing: {error}"))?;
+ }
+
+ Ok(Observability { tracer_provider })
+}
+
+fn build_tracer_provider() -> Result<SdkTracerProvider, String> {
+ let exporter = opentelemetry_otlp::SpanExporter::builder()
+ .with_http()
+ .build()
+ .map_err(|error| format!("failed to initialize OTLP trace exporter: {error}"))?;
+ let provider = SdkTracerProvider::builder()
+ .with_resource(resource())
+ .with_batch_exporter(exporter)
+ .build();
+ opentelemetry::global::set_tracer_provider(provider.clone());
+ Ok(provider)
+}
+
+fn env_var_is_true(name: &str) -> bool {
+ env::var(name).is_ok_and(|value| matches!(value.to_ascii_lowercase().as_str(), "true" | "1"))
+}
diff --git a/switchyard_rust/server.py b/switchyard_rust/server.py
new file mode 100644
index 000000000..46b5d6036
--- /dev/null
+++ b/switchyard_rust/server.py
@@ -0,0 +1,46 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Native Rust Switchyard server host."""
+
+from __future__ import annotations
+
+from os import PathLike
+from typing import TYPE_CHECKING, Any, final
+
+from switchyard_rust.core import _load_native
+
+if TYPE_CHECKING:
+
+ @final
+ class Server:
+ """Running loopback instance of the native Switchyard server."""
+
+ def __init__(self, config: str | PathLike[str], *, port: int = 0) -> None: ...
+
+ @property
+ def port(self) -> int: ...
+
+ @property
+ def base_url(self) -> str: ...
+
+ def close(self, *, timeout_secs: float = 2.0) -> None: ...
+
+ def __enter__(self) -> Server: ...
+
+ def __exit__(
+ self,
+ exception_type: type[BaseException] | None,
+ exception: BaseException | None,
+ traceback: object | None,
+ ) -> bool: ...
+
+
+def __getattr__(name: str) -> object:
+ if name == "Server":
+ native: Any = _load_native()
+ return native.server.Server
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = ["Server"]