Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(server): add native Python host with OpenTelemetry (#210)
nachiketb-nvidia50e8d74dbead
diff --git a/Cargo.lock b/Cargo.lockindex 5cf8b253d..ab0d4052a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -61,6 +61,12 @@ dependencies = ["windows-sys 0.61.2",]+[[package]]+name = "anyhow"+version = "1.0.104"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"+[[package]]name = "arc-swap"version = "1.9.2"@@ -336,6 +342,32 @@ version = "1.0.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"+[[package]]+name = "combine"+version = "4.6.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd"+dependencies = [+ "bytes",+ "memchr",+]++[[package]]+name = "core-foundation"+version = "0.10.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"+dependencies = [+ "core-foundation-sys",+ "libc",+]++[[package]]+name = "core-foundation-sys"+version = "0.8.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"+[[package]]name = "cpufeatures"version = "0.3.0"@@ -846,12 +878,70 @@ version = "1.70.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"+[[package]]+name = "itertools"+version = "0.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"+dependencies = [+ "either",+]+[[package]]name = "itoa"version = "1.0.18"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"+[[package]]+name = "jni"+version = "0.22.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498"+dependencies = [+ "cfg-if",+ "combine",+ "jni-macros",+ "jni-sys",+ "log",+ "simd_cesu8",+ "thiserror 2.0.18",+ "walkdir",+ "windows-link",+]++[[package]]+name = "jni-macros"+version = "0.22.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3"+dependencies = [+ "proc-macro2",+ "quote",+ "rustc_version",+ "simd_cesu8",+ "syn",+]++[[package]]+name = "jni-sys"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"+dependencies = [+ "jni-sys-macros",+]++[[package]]+name = "jni-sys-macros"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"+dependencies = [+ "quote",+ "syn",+]+[[package]]name = "jobserver"version = "0.1.35"@@ -987,6 +1077,12 @@ version = "1.70.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"+[[package]]+name = "openssl-probe"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"+[[package]]name = "opentelemetry"version = "0.32.0"@@ -1001,6 +1097,35 @@ dependencies = ["tracing",]+[[package]]+name = "opentelemetry-http"+version = "0.32.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331"+dependencies = [+ "async-trait",+ "bytes",+ "http",+ "opentelemetry",+ "reqwest 0.13.4",+]++[[package]]+name = "opentelemetry-otlp"+version = "0.32.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35"+dependencies = [+ "http",+ "opentelemetry",+ "opentelemetry-http",+ "opentelemetry-proto",+ "opentelemetry_sdk",+ "prost",+ "reqwest 0.13.4",+ "thiserror 2.0.18",+]+[[package]]name = "opentelemetry-prometheus"version = "0.32.0"@@ -1014,6 +1139,17 @@ dependencies = ["tracing",]+[[package]]+name = "opentelemetry-proto"+version = "0.32.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638"+dependencies = [+ "opentelemetry",+ "opentelemetry_sdk",+ "prost",+]+[[package]]name = "opentelemetry_sdk"version = "0.32.1"@@ -1130,6 +1266,29 @@ dependencies = ["thiserror 2.0.18",]+[[package]]+name = "prost"+version = "0.14.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1"+dependencies = [+ "bytes",+ "prost-derive",+]++[[package]]+name = "prost-derive"+version = "0.14.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"+dependencies = [+ "anyhow",+ "itertools",+ "proc-macro2",+ "quote",+ "syn",+]+[[package]]name = "protobuf"version = "3.7.2"@@ -1258,6 +1417,7 @@ version = "0.11.16"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560"dependencies = [+ "aws-lc-rs","bytes","getrandom 0.4.3","lru-slab",@@ -1473,6 +1633,43 @@ dependencies = ["webpki-roots",]+[[package]]+name = "reqwest"+version = "0.13.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"+dependencies = [+ "base64",+ "bytes",+ "futures-channel",+ "futures-core",+ "futures-util",+ "http",+ "http-body",+ "http-body-util",+ "hyper",+ "hyper-rustls",+ "hyper-util",+ "js-sys",+ "log",+ "percent-encoding",+ "pin-project-lite",+ "quinn",+ "rustls",+ "rustls-pki-types",+ "rustls-platform-verifier",+ "sync_wrapper",+ "tokio",+ "tokio-rustls",+ "tower",+ "tower-http",+ "tower-service",+ "url",+ "wasm-bindgen",+ "wasm-bindgen-futures",+ "web-sys",+]+[[package]]name = "ring"version = "0.17.14"@@ -1493,6 +1690,15 @@ version = "2.1.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"+[[package]]+name = "rustc_version"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"+dependencies = [+ "semver",+]+[[package]]name = "rustix"version = "1.1.4"@@ -1521,6 +1727,18 @@ dependencies = ["zeroize",]+[[package]]+name = "rustls-native-certs"+version = "0.8.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d"+dependencies = [+ "openssl-probe",+ "rustls-pki-types",+ "schannel",+ "security-framework",+]+[[package]]name = "rustls-pki-types"version = "1.15.0"@@ -1531,6 +1749,33 @@ dependencies = ["zeroize",]+[[package]]+name = "rustls-platform-verifier"+version = "0.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0"+dependencies = [+ "core-foundation",+ "core-foundation-sys",+ "jni",+ "log",+ "once_cell",+ "rustls",+ "rustls-native-certs",+ "rustls-platform-verifier-android",+ "rustls-webpki",+ "security-framework",+ "security-framework-sys",+ "webpki-root-certs",+ "windows-sys 0.61.2",+]++[[package]]+name = "rustls-platform-verifier-android"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"+[[package]]name = "rustls-webpki"version = "0.103.13"@@ -1555,12 +1800,59 @@ version = "1.0.23"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"+[[package]]+name = "same-file"+version = "1.0.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"+dependencies = [+ "winapi-util",+]++[[package]]+name = "schannel"+version = "0.1.29"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"+dependencies = [+ "windows-sys 0.61.2",+]+[[package]]name = "scopeguard"version = "1.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"+[[package]]+name = "security-framework"+version = "3.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"+dependencies = [+ "bitflags",+ "core-foundation",+ "core-foundation-sys",+ "libc",+ "security-framework-sys",+]++[[package]]+name = "security-framework-sys"+version = "2.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3"+dependencies = [+ "core-foundation-sys",+ "libc",+]++[[package]]+name = "semver"+version = "1.0.28"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"+[[package]]name = "serde"version = "1.0.228"@@ -1661,6 +1953,22 @@ dependencies = ["libc",]+[[package]]+name = "simd_cesu8"+version = "1.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520"+dependencies = [+ "rustc_version",+ "simdutf8",+]++[[package]]+name = "simdutf8"+version = "0.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"+[[package]]name = "slab"version = "0.4.12"@@ -1711,7 +2019,7 @@ dependencies = ["futures-util","parking_lot","rand 0.8.7",- "reqwest",+ "reqwest 0.12.28","serde","serde_json","switchyard-libsy",@@ -1752,7 +2060,7 @@ dependencies = ["futures-util","httpdate","opentelemetry",- "reqwest",+ "reqwest 0.12.28","serde_json","switchyard-protocol","switchyard-translation",@@ -1787,6 +2095,7 @@ dependencies = ["switchyard-components","switchyard-libsy","switchyard-protocol",+ "switchyard-server","switchyard-translation","tokio","tracing",@@ -1804,6 +2113,7 @@ dependencies = ["futures-util","http-body-util","opentelemetry",+ "opentelemetry-otlp","opentelemetry-prometheus","opentelemetry_sdk","parking_lot",@@ -1819,6 +2129,7 @@ dependencies = ["toml","tower","tracing",+ "tracing-opentelemetry","tracing-subscriber",]@@ -2161,6 +2472,22 @@ dependencies = ["tracing-core",]+[[package]]+name = "tracing-opentelemetry"+version = "0.33.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26"+dependencies = [+ "js-sys",+ "opentelemetry",+ "smallvec",+ "tracing",+ "tracing-core",+ "tracing-log",+ "tracing-subscriber",+ "web-time",+]+[[package]]name = "tracing-subscriber"version = "0.3.23"@@ -2227,6 +2554,16 @@ version = "0.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"+[[package]]+name = "walkdir"+version = "2.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"+dependencies = [+ "same-file",+ "winapi-util",+]+[[package]]name = "want"version = "0.3.1"@@ -2339,6 +2676,15 @@ dependencies = ["wasm-bindgen",]+[[package]]+name = "webpki-root-certs"+version = "1.0.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"+dependencies = [+ "rustls-pki-types",+]+[[package]]name = "webpki-roots"version = "1.0.9"@@ -2348,6 +2694,15 @@ dependencies = ["rustls-pki-types",]+[[package]]+name = "winapi-util"+version = "0.1.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"+dependencies = [+ "windows-sys 0.61.2",+]+[[package]]name = "windows-link"version = "0.2.1"diff --git a/Cargo.toml b/Cargo.tomlindex 7fd0c69cf..8bf001d3d 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -36,6 +36,7 @@ switchyard-components = { path = "crates/switchyard-components", version = "0.1.switchyard-libsy = { path = "crates/libsy", version = "0.1.0" }switchyard-llm-client = { path = "crates/libsy-llm-client", version = "0.1.0" }switchyard-protocol = { path = "crates/protocol", version = "0.1.0" }+switchyard-server = { path = "crates/switchyard-server", version = "0.1.0" }switchyard-translation = { path = "crates/switchyard-translation", version = "0.1.0" }thiserror = "2"tokio = { version = "1", features = ["full"] }diff --git a/crates/switchyard-py/Cargo.toml b/crates/switchyard-py/Cargo.tomlindex 90d2e76f5..56dd7e3c2 100644--- a/crates/switchyard-py/Cargo.toml+++ b/crates/switchyard-py/Cargo.toml@@ -26,6 +26,7 @@ pythonize = "0.28.0"serde.workspace = trueserde_json.workspace = trueswitchyard-protocol.workspace = true+switchyard-server.workspace = trueswitchyard-components.workspace = trueswitchyard-translation.workspace = truetokio.workspace = truediff --git a/crates/switchyard-py/src/lib.rs b/crates/switchyard-py/src/lib.rsindex f9862c69f..3082697a1 100644--- a/crates/switchyard-py/src/lib.rs+++ b/crates/switchyard-py/src/lib.rs@@ -8,6 +8,7 @@ mod errors;mod interop;mod libsy_bindings;mod py_serde;+mod server_bindings;mod translation;#[pymodule]@@ -17,5 +18,6 @@ fn _switchyard_rust(module: &Bound<'_, PyModule>) -> PyResult<()> {interop::register(module)?;component_bindings::register(module)?;libsy_bindings::register(module)?;+ server_bindings::register(module)?;Ok(())}diff --git a/crates/switchyard-py/src/server_bindings.rs b/crates/switchyard-py/src/server_bindings.rsnew file mode 100644index 000000000..63caffe9f--- /dev/null+++ b/crates/switchyard-py/src/server_bindings.rs@@ -0,0 +1,162 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Self-contained Python host for the Rust Switchyard server.++use std::net::{IpAddr, Ipv4Addr, SocketAddr};+use std::path::PathBuf;+use std::sync::mpsc::{sync_channel, Receiver, RecvTimeoutError};+use std::time::Duration;++use pyo3::exceptions::{PyRuntimeError, PyValueError};+use pyo3::prelude::*;+use switchyard_server::config::load_server_state;+use switchyard_server::{+ flush_observability, initialize_observability, BoundServer, ServerResult, ServerRunOptions,+ DEFAULT_LISTEN_BACKLOG,+};+use tokio::sync::oneshot;+use tokio::task::JoinHandle;++const DEFAULT_SHUTDOWN_TIMEOUT_SECS: f64 = 2.0;++/// Running loopback server backed entirely by the native Rust implementation.+#[pyclass(name = "Server", module = "switchyard_rust.server", unsendable)]+struct PyServer {+ addr: SocketAddr,+ shutdown: Option<oneshot::Sender<()>>,+ completion: Option<Receiver<ServerResult<()>>>,+ task: Option<JoinHandle<()>>,+}++#[pymethods]+impl PyServer {+ /// Loads a TOML deployment and starts serving it on loopback.+ #[new]+ #[pyo3(signature = (config, *, port=0))]+ fn new(config: PathBuf, port: u16) -> PyResult<Self> {+ initialize_observability().map_err(server_error)?;+ let state = load_server_state(config).map_err(server_error)?;+ let runtime = pyo3_async_runtimes::tokio::get_runtime();+ let server = {+ let _guard = runtime.enter();+ BoundServer::bind(+ state,+ ServerRunOptions {+ addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port),+ backlog: DEFAULT_LISTEN_BACKLOG,+ dry_run: false,+ tls: None,+ },+ )+ .map_err(server_error)?+ };+ let addr = server.local_addr();+ let (shutdown, shutdown_receiver) = oneshot::channel();+ let (completion_sender, completion) = sync_channel(1);+ let task = runtime.spawn(async move {+ let result = server+ .serve(async move {+ let _ = shutdown_receiver.await;+ })+ .await;+ let _ = completion_sender.send(result);+ });+ Ok(Self {+ addr,+ shutdown: Some(shutdown),+ completion: Some(completion),+ task: Some(task),+ })+ }++ /// Returns the bound loopback port.+ #[getter]+ fn port(&self) -> u16 {+ self.addr.port()+ }++ /// Returns the HTTP base URL for clients.+ #[getter]+ fn base_url(&self) -> String {+ format!("http://{}", self.addr)+ }++ /// Gracefully stops the server and flushes pending telemetry.+ #[pyo3(signature = (*, timeout_secs=DEFAULT_SHUTDOWN_TIMEOUT_SECS))]+ fn close(&mut self, py: Python<'_>, timeout_secs: f64) -> PyResult<()> {+ self.close_inner(py, timeout_secs)+ }++ fn __enter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> {+ slf+ }++ fn __exit__(+ &mut self,+ py: Python<'_>,+ _exception_type: &Bound<'_, PyAny>,+ _exception: &Bound<'_, PyAny>,+ _traceback: &Bound<'_, PyAny>,+ ) -> PyResult<bool> {+ self.close_inner(py, DEFAULT_SHUTDOWN_TIMEOUT_SECS)?;+ Ok(false)+ }++ fn __repr__(&self) -> String {+ format!("Server(base_url={:?})", self.base_url())+ }+}++impl PyServer {+ fn close_inner(&mut self, py: Python<'_>, timeout_secs: f64) -> PyResult<()> {+ let timeout = Duration::try_from_secs_f64(timeout_secs).map_err(|_| {+ PyValueError::new_err("timeout_secs must be a supported finite non-negative number")+ })?;+ if let Some(shutdown) = self.shutdown.take() {+ let _ = shutdown.send(());+ }+ let Some(completion) = self.completion.take() else {+ return Ok(());+ };+ let task = self.task.take();+ let result = py.detach(move || {+ let result = completion.recv_timeout(timeout);+ if matches!(result, Err(RecvTimeoutError::Timeout)) {+ if let Some(task) = task {+ task.abort();+ }+ }+ flush_observability();+ result+ });+ match result {+ Ok(result) => result.map_err(server_error),+ Err(RecvTimeoutError::Timeout) => Err(PyRuntimeError::new_err(format!(+ "server did not stop within {timeout_secs} seconds"+ ))),+ Err(RecvTimeoutError::Disconnected) => Err(PyRuntimeError::new_err(+ "server task stopped without reporting a result",+ )),+ }+ }+}++impl Drop for PyServer {+ fn drop(&mut self) {+ if let Some(shutdown) = self.shutdown.take() {+ let _ = shutdown.send(());+ }+ }+}++fn server_error(error: impl std::fmt::Display) -> PyErr {+ PyRuntimeError::new_err(error.to_string())+}++pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {+ let server_module = PyModule::new(module.py(), "server")?;+ server_module.add_class::<PyServer>()?;+ module.add_submodule(&server_module)?;+ Ok(())+}diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.tomlindex 424d73a5f..119c311b9 100644--- a/crates/switchyard-server/Cargo.toml+++ b/crates/switchyard-server/Cargo.toml@@ -20,10 +20,11 @@ rustls = { version = "0.23", default-features = false, features = ["aws-lc-rs",clap = { version = "4", features = ["derive", "env"] }futures-util.workspace = truelibsy = { package = "switchyard-libsy", path = "../libsy" }-opentelemetry = { version = "0.32", default-features = false, features = ["metrics"] }+opentelemetry = { version = "0.32", default-features = false, features = ["metrics", "trace"] }+opentelemetry-otlp = { version = "0.32", default-features = false, features = ["http-proto", "metrics", "reqwest-blocking-client", "reqwest-rustls", "trace"] }opentelemetry-prometheus = "0.32"-opentelemetry_sdk = { version = "0.32", default-features = false, features = ["metrics"] }parking_lot.workspace = true+opentelemetry_sdk = { version = "0.32", default-features = false, features = ["metrics", "trace"] }prometheus = "0.14"serde.workspace = truetoml = "1.1"@@ -32,6 +33,7 @@ serde_json.workspace = trueswitchyard-translation.workspace = truetokio.workspace = truetracing.workspace = true+tracing-opentelemetry = "0.33"tracing-subscriber = { version = "0.3", features = ["env-filter"] }[dev-dependencies]diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex db84554db..eb0a8a898 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -5,6 +5,7 @@pub mod config;mod metrics;+mod observability;mod response;mod sse;mod stats;@@ -13,6 +14,7 @@ mod usage_metrics;use std::collections::BTreeMap;use std::error::Error;use std::fmt::{Display, Formatter};+use std::future::Future;use std::net::SocketAddr;use std::path::PathBuf;use std::sync::Arc;@@ -39,6 +41,8 @@ use switchyard_translation::{decode_request, WireFormat};use crate::response::into_http_response;use crate::stats::{prefix_probe, tracking_enabled_from_env, StatsAccumulator, StatsSnapshot};+pub use observability::{flush_observability, initialize_observability};+/// Default TCP listen backlog used by the Rust server.pub const DEFAULT_LISTEN_BACKLOG: u32 = 65_535;@@ -156,22 +160,60 @@ pub async fn run_server(state: ServerState, options: ServerRunOptions) -> Serverreturn Ok(());}- let listener = bind_tcp_listener(options.addr, options.backlog)?;- let bound_addr = listener.local_addr().map_err(server_io_error)?;- let server_options = ServerRunOptions {- addr: bound_addr,- ..options- };- eprintln!("{}", startup_banner(&server_options, &state));- let router = build_switchyard_router(state);- if let Some(tls) = server_options.tls {- serve_tls(listener, router, tls).await- } else {- serve(listener, router).await+ let server = BoundServer::bind(state, options)?;+ eprintln!("{}", server.startup_banner());+ server.serve(shutdown_signal()).await+}++/// A configured server with its listening socket already bound.+pub struct BoundServer {+ listener: TcpListener,+ router: Router,+ options: ServerRunOptions,+ state: ServerState,+}++impl BoundServer {+ /// Binds the configured address and prepares the HTTP router.+ pub fn bind(state: ServerState, options: ServerRunOptions) -> ServerResult<Self> {+ let listener = bind_tcp_listener(options.addr, options.backlog)?;+ let addr = listener.local_addr().map_err(server_io_error)?;+ Ok(Self {+ listener,+ router: build_switchyard_router(state.clone()),+ options: ServerRunOptions { addr, ..options },+ state,+ })+ }++ /// Returns the actual bound address, including an OS-selected port.+ pub fn local_addr(&self) -> SocketAddr {+ self.options.addr+ }++ /// Serves requests until the supplied shutdown future resolves.+ pub async fn serve(+ self,+ shutdown: impl Future<Output = ()> + Send + 'static,+ ) -> ServerResult<()> {+ if let Some(tls) = self.options.tls {+ serve_tls(self.listener, self.router, tls, shutdown).await+ } else {+ serve(self.listener, self.router, shutdown).await+ }+ }++ fn startup_banner(&self) -> String {+ startup_banner(&self.options, &self.state)}}-async fn serve_tls(listener: TcpListener, router: Router, tls: TlsOptions) -> ServerResult<()> {+async fn serve_tls(+ listener: TcpListener,+ router: Router,+ tls: TlsOptions,+ shutdown: impl Future<Output = ()> + Send + 'static,+) -> ServerResult<()> {if let Err(error) = rustls::crypto::aws_lc_rs::default_provider().install_default() {tracing::debug!(?error, "TLS crypto provider was already installed");}@@ -183,7 +225,7 @@ async fn serve_tls(listener: TcpListener, router: Router, tls: TlsOptions) -> Selet shutdown_handle = handle.clone();tokio::spawn(async move {- shutdown_signal().await;+ shutdown.await;shutdown_handle.graceful_shutdown(Some(Duration::from_secs(2)));});@@ -196,9 +238,13 @@ async fn serve_tls(listener: TcpListener, router: Router, tls: TlsOptions) -> Se.map_err(server_io_error)}-async fn serve(listener: TcpListener, router: Router) -> ServerResult<()> {+async fn serve(+ listener: TcpListener,+ router: Router,+ shutdown: impl Future<Output = ()> + Send + 'static,+) -> ServerResult<()> {axum::serve(listener, router)- .with_graceful_shutdown(shutdown_signal())+ .with_graceful_shutdown(shutdown).await.map_err(server_io_error)}diff --git a/crates/switchyard-server/src/main.rs b/crates/switchyard-server/src/main.rsindex 51c44a4be..b9f7fdf0a 100644--- a/crates/switchyard-server/src/main.rs+++ b/crates/switchyard-server/src/main.rs@@ -5,36 +5,21 @@use std::process::ExitCode;-use tracing_subscriber::EnvFilter;-mod cli;-const DEFAULT_LOG_FILTER: &str = "switchyard_server=info";-const OPENTELEMETRY_LOG_FILTER: &str = "opentelemetry=warn";-#[tokio::main(flavor = "multi_thread")]async fn main() -> ExitCode {- if let Err(error) = init_logging() {- eprintln!("failed to initialize logging: {error}");+ if let Err(error) = switchyard_server::initialize_observability() {+ eprintln!("failed to initialize observability: {error}");return ExitCode::FAILURE;}- match cli::run(cli::ServerArgs::parse_args()).await {+ let exit_code = match cli::run(cli::ServerArgs::parse_args()).await {Ok(()) => ExitCode::SUCCESS,Err(error) => {eprintln!("{error}");ExitCode::FAILURE}- }-}--fn init_logging() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {- let filter = EnvFilter::try_from_default_env()- .unwrap_or_else(|_| EnvFilter::new(DEFAULT_LOG_FILTER))- .add_directive(OPENTELEMETRY_LOG_FILTER.parse()?);- tracing_subscriber::fmt()- .with_env_filter(filter)- .with_ansi(false)- .with_writer(std::io::stderr)- .try_init()?;- Ok(())+ };+ switchyard_server::flush_observability();+ exit_code}diff --git a/crates/switchyard-server/src/metrics.rs b/crates/switchyard-server/src/metrics.rsindex 8cd3df141..8be2148dc 100644--- a/crates/switchyard-server/src/metrics.rs+++ b/crates/switchyard-server/src/metrics.rs@@ -40,10 +40,18 @@ fn initialize() -> Result<Metrics, String> {.with_registry(registry.clone()).build().map_err(|error| format!("failed to initialize Prometheus metrics: {error}"))?;- let provider = SdkMeterProvider::builder()+ let mut builder = SdkMeterProvider::builder().with_reader(exporter).with_view(routing_overhead_buckets)- .build();+ .with_resource(crate::observability::resource());+ if crate::observability::otlp_enabled("METRICS") {+ let exporter = opentelemetry_otlp::MetricExporter::builder()+ .with_http()+ .build()+ .map_err(|error| format!("failed to initialize OTLP metric exporter: {error}"))?;+ builder = builder.with_periodic_exporter(exporter);+ }+ let provider = builder.build();global::set_meter_provider(provider.clone());libsy::initialize_metrics();global::meter("switchyard")@@ -57,6 +65,14 @@ fn initialize() -> Result<Metrics, String> {})}+pub(crate) fn flush() {+ if let Some(Ok(metrics)) = METRICS.get() {+ if let Err(error) = metrics._provider.force_flush() {+ tracing::warn!(error = %error, "failed to flush OpenTelemetry metrics");+ }+ }+}+fn routing_overhead_buckets(instrument: &Instrument) -> Option<Stream> {if instrument.name() != "switchyard.routing_overhead_ms" {return None;diff --git a/crates/switchyard-server/src/observability.rs b/crates/switchyard-server/src/observability.rsnew file mode 100644index 000000000..5c76ec3dd--- /dev/null+++ b/crates/switchyard-server/src/observability.rs@@ -0,0 +1,134 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Process-wide tracing and OpenTelemetry setup for server hosts.++use std::env;+use std::sync::OnceLock;++use opentelemetry::trace::TracerProvider as _;+use opentelemetry_sdk::trace::SdkTracerProvider;+use opentelemetry_sdk::Resource;+use tracing_subscriber::layer::SubscriberExt as _;+use tracing_subscriber::util::SubscriberInitExt as _;+use tracing_subscriber::{EnvFilter, Layer as _};++use crate::{metrics, ServerError, ServerResult};++const DEFAULT_LOG_FILTER: &str = "switchyard_server=info";+const OPENTELEMETRY_LOG_FILTER: &str = "opentelemetry=warn";+const OTEL_SPAN_FILTER: &str = "libsy=info";+const DEFAULT_SERVICE_NAME: &str = "switchyard-server";++struct Observability {+ tracer_provider: Option<SdkTracerProvider>,+}++static OBSERVABILITY: OnceLock<Result<Observability, String>> = OnceLock::new();++/// Installs metrics and tracing once for either the binary or an embedded host.+pub fn initialize_observability() -> ServerResult<()> {+ match OBSERVABILITY.get_or_init(initialize) {+ Ok(_) => Ok(()),+ Err(error) => Err(ServerError::new(error.clone())),+ }+}++/// Flushes pending OTLP telemetry without shutting down process-wide providers.+pub fn flush_observability() {+ if let Some(Ok(observability)) = OBSERVABILITY.get() {+ if let Some(provider) = &observability.tracer_provider {+ if let Err(error) = provider.force_flush() {+ tracing::warn!(error = %error, "failed to flush OpenTelemetry traces");+ }+ }+ }+ metrics::flush();+}++pub(crate) fn otlp_enabled(signal: &str) -> bool {+ if env_var_is_true("OTEL_SDK_DISABLED") {+ return false;+ }+ if env::var(format!("OTEL_{signal}_EXPORTER"))+ .ok()+ .filter(|value| !value.trim().is_empty())+ .is_some_and(|value| {+ !value+ .split(',')+ .any(|exporter| exporter.trim().eq_ignore_ascii_case("otlp"))+ })+ {+ return false;+ }+ [+ "OTEL_EXPORTER_OTLP_ENDPOINT",+ &format!("OTEL_EXPORTER_OTLP_{signal}_ENDPOINT"),+ ]+ .into_iter()+ .any(|name| env::var(name).is_ok_and(|value| !value.trim().is_empty()))+}++pub(crate) fn resource() -> Resource {+ let service_name = env::var("OTEL_SERVICE_NAME")+ .ok()+ .filter(|value| !value.trim().is_empty())+ .unwrap_or_else(|| DEFAULT_SERVICE_NAME.to_string());+ Resource::builder().with_service_name(service_name).build()+}++fn initialize() -> Result<Observability, String> {+ metrics::registry()?;++ let tracer_provider = otlp_enabled("TRACES")+ .then(build_tracer_provider)+ .transpose()?;+ let filter = EnvFilter::try_from_default_env()+ .unwrap_or_else(|_| EnvFilter::new(DEFAULT_LOG_FILTER))+ .add_directive(+ OPENTELEMETRY_LOG_FILTER+ .parse()+ .map_err(|error| format!("invalid OpenTelemetry log filter: {error}"))?,+ );+ let format = tracing_subscriber::fmt::layer()+ .with_ansi(false)+ .with_writer(std::io::stderr)+ .with_filter(filter);++ if let Some(provider) = &tracer_provider {+ let tracer = provider.tracer("switchyard");+ tracing_subscriber::registry()+ .with(format)+ .with(+ tracing_opentelemetry::layer()+ .with_tracer(tracer)+ .with_filter(EnvFilter::new(OTEL_SPAN_FILTER)),+ )+ .try_init()+ .map_err(|error| format!("failed to initialize tracing: {error}"))?;+ } else {+ tracing_subscriber::registry()+ .with(format)+ .try_init()+ .map_err(|error| format!("failed to initialize tracing: {error}"))?;+ }++ Ok(Observability { tracer_provider })+}++fn build_tracer_provider() -> Result<SdkTracerProvider, String> {+ let exporter = opentelemetry_otlp::SpanExporter::builder()+ .with_http()+ .build()+ .map_err(|error| format!("failed to initialize OTLP trace exporter: {error}"))?;+ let provider = SdkTracerProvider::builder()+ .with_resource(resource())+ .with_batch_exporter(exporter)+ .build();+ opentelemetry::global::set_tracer_provider(provider.clone());+ Ok(provider)+}++fn env_var_is_true(name: &str) -> bool {+ env::var(name).is_ok_and(|value| matches!(value.to_ascii_lowercase().as_str(), "true" | "1"))+}diff --git a/switchyard_rust/server.py b/switchyard_rust/server.pynew file mode 100644index 000000000..46b5d6036--- /dev/null+++ b/switchyard_rust/server.py@@ -0,0 +1,46 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Native Rust Switchyard server host."""++from __future__ import annotations++from os import PathLike+from typing import TYPE_CHECKING, Any, final++from switchyard_rust.core import _load_native++if TYPE_CHECKING:++ @final+ class Server:+ """Running loopback instance of the native Switchyard server."""++ def __init__(self, config: str | PathLike[str], *, port: int = 0) -> None: ...++ @property+ def port(self) -> int: ...++ @property+ def base_url(self) -> str: ...++ def close(self, *, timeout_secs: float = 2.0) -> None: ...++ def __enter__(self) -> Server: ...++ def __exit__(+ self,+ exception_type: type[BaseException] | None,+ exception: BaseException | None,+ traceback: object | None,+ ) -> bool: ...+++def __getattr__(name: str) -> object:+ if name == "Server":+ native: Any = _load_native()+ return native.server.Server+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")+++__all__ = ["Server"]