Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
refactor(server): serve libsy algorithms without profiles (#98)
nachiketb-nvidiae09b5b032d5f
diff --git a/Cargo.lock b/Cargo.lockindex 0370a7e5e..f45b55c22 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1747,7 +1747,6 @@ dependencies = ["switchyard-components-v2","switchyard-core","switchyard-libsy",- "switchyard-server","switchyard-translation","tokio","tracing",@@ -1765,10 +1764,9 @@ dependencies = ["futures-util","http-body-util","rustls",- "serde","serde_json",- "switchyard-components-v2",- "switchyard-core",+ "switchyard-libsy",+ "switchyard-llm-client","switchyard-translation","tokio","tower",diff --git a/crates/switchyard-py/Cargo.toml b/crates/switchyard-py/Cargo.tomlindex 074625a67..560ea2f65 100644--- a/crates/switchyard-py/Cargo.toml+++ b/crates/switchyard-py/Cargo.toml@@ -27,7 +27,6 @@ serde_json = "1"switchyard-core = { path = "../switchyard-core" }switchyard-components = { path = "../switchyard-components" }switchyard-components-v2 = { path = "../switchyard-components-v2" }-switchyard-server = { path = "../switchyard-server" }switchyard-translation = { path = "../switchyard-translation" }tokio = { version = "1", features = ["rt", "rt-multi-thread", "sync"] }tracing = { version = "0.1", default-features = false, features = ["std"] }diff --git a/crates/switchyard-py/src/lib.rs b/crates/switchyard-py/src/lib.rsindex 6b3fe73d3..5ef43c27a 100644--- a/crates/switchyard-py/src/lib.rs+++ b/crates/switchyard-py/src/lib.rs@@ -9,7 +9,6 @@ mod errors;mod libsy_bindings;mod profile_bindings;mod py_serde;-mod server_bindings;mod translation;#[pymodule]@@ -20,6 +19,5 @@ fn _switchyard_rust(module: &Bound<'_, PyModule>) -> PyResult<()> {component_bindings::register(module)?;libsy_bindings::register(module)?;profile_bindings::register(module)?;- server_bindings::register(module)?;Ok(())}diff --git a/crates/switchyard-py/src/server_bindings.rs b/crates/switchyard-py/src/server_bindings.rsdeleted file mode 100644index a1b5cec87..000000000--- a/crates/switchyard-py/src/server_bindings.rs+++ /dev/null@@ -1,60 +0,0 @@-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-// SPDX-License-Identifier: Apache-2.0--//! Python bindings for the Rust components-v2 profile server.--use std::net::{IpAddr, SocketAddr};-use std::path::PathBuf;--use pyo3::exceptions::PyValueError;-use pyo3::prelude::*;-use switchyard_server::{run_server, ServerRunOptions, DEFAULT_LISTEN_BACKLOG};--use crate::errors::py_core_error;--/// Run the Rust components-v2 profile server from Python.-#[pyfunction]-#[pyo3(signature = (- config_path,- host = "127.0.0.1",- port = 4000,- backlog = DEFAULT_LISTEN_BACKLOG,- dry_run = false,-))]-fn run_profile_server(- py: Python<'_>,- config_path: String,- host: &str,- port: u16,- backlog: u32,- dry_run: bool,-) -> PyResult<()> {- let ip: IpAddr = host.parse().map_err(|error| {- PyValueError::new_err(format!(- "host must be an IP address accepted by the Rust server, got {host:?}: {error}"- ))- })?;- let options = ServerRunOptions {- config: PathBuf::from(config_path),- addr: SocketAddr::new(ip, port),- backlog,- dry_run,- tls: None,- };-- // `detach` runs synchronously with the GIL released, so startup errors still- // return to the Python caller instead of disappearing into a background task.- py.detach(move || {- let runtime = tokio::runtime::Builder::new_multi_thread()- .enable_all()- .build()- .map_err(|error| PyValueError::new_err(error.to_string()))?;- runtime.block_on(run_server(options)).map_err(py_core_error)- })-}--/// Registers Rust server bindings with the native Python module.-pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {- module.add_function(wrap_pyfunction!(run_profile_server, module)?)?;- Ok(())-}diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.tomlindex 23012b853..88e88ac02 100644--- a/crates/switchyard-server/Cargo.toml+++ b/crates/switchyard-server/Cargo.toml@@ -4,7 +4,7 @@[package]name = "switchyard-server"version = "0.1.0"-description = "Rust HTTP server surface for Switchyard profiles"+description = "Rust HTTP server surface for libsy algorithms"authors.workspace = trueedition.workspace = truelicense.workspace = true@@ -18,10 +18,9 @@ axum-server = { version = "0.8", features = ["tls-rustls"] }rustls = { version = "0.23" }clap = { version = "4", features = ["derive", "env"] }futures-util = "0.3"-serde = { version = "1", features = ["derive"] }+libsy = { package = "switchyard-libsy", path = "../libsy" }+switchyard-llm-client = { path = "../libsy-llm-client" }serde_json = "1"-switchyard-components-v2 = { path = "../switchyard-components-v2" }-switchyard-core = { path = "../switchyard-core" }switchyard-translation = { path = "../switchyard-translation" }tokio = { version = "1", features = ["macros", "net", "rt-multi-thread", "signal"] }tracing = { version = "0.1", default-features = false, features = ["std"] }diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.mdnew file mode 100644index 000000000..4bed0fd07--- /dev/null+++ b/crates/switchyard-server/README.md@@ -0,0 +1,15 @@+# switchyard-server++`switchyard-server` exposes libsy algorithms through OpenAI Chat Completions, OpenAI Responses,+and Anthropic Messages endpoints. The current CLI builds uniform-random routes over one shared+upstream backend.++```bash+cargo run -p switchyard-server -- \+ --route switchyard/general=model/a,model/b \+ --route switchyard/coding=model/c,model/d \+ --base-url https://example.com/v1 \+ --api-key "$API_KEY"+```++Select a route by sending its `switchyard/*` ID as the request's `model`.diff --git a/crates/switchyard-server/src/cli.rs b/crates/switchyard-server/src/cli.rsindex d71008ce6..63e88345e 100644--- a/crates/switchyard-server/src/cli.rs+++ b/crates/switchyard-server/src/cli.rs@@ -1,53 +1,132 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-//! CLI entrypoint for running the components-v2 Rust profile server.+//! CLI entrypoint for running the libsy server with random routing.+use std::collections::{BTreeMap, BTreeSet};use std::net::{IpAddr, Ipv4Addr, SocketAddr};use std::path::PathBuf;+use std::str::FromStr;+use std::sync::Arc;-use clap::Parser;-use switchyard_core::{Result, SwitchyardError};-use switchyard_server::{run_server, ServerRunOptions, TLSOptions, DEFAULT_LISTEN_BACKLOG};+use clap::{Parser, ValueEnum};+use libsy::algorithms::Random;+use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient};+use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};+use switchyard_server::{+ run_server, ServerError, ServerResult, ServerRunOptions, ServerState, TlsOptions,+ DEFAULT_LISTEN_BACKLOG,+};const DEFAULT_HOST: IpAddr = IpAddr::V4(Ipv4Addr::UNSPECIFIED);const DEFAULT_PORT: u16 = 4000;+#[derive(Clone, Debug)]+struct RandomRouteSpec {+ model: String,+ targets: Vec<String>,+}++impl FromStr for RandomRouteSpec {+ type Err = String;++ fn from_str(value: &str) -> Result<Self, Self::Err> {+ let (model, target_list) = value+ .split_once('=')+ .ok_or_else(|| "route must use MODEL=TARGET[,TARGET...]".to_string())?;+ let model = model.trim();+ if model.is_empty() {+ return Err("route model must not be empty".to_string());+ }+ let targets = target_list+ .split(',')+ .map(str::trim)+ .map(str::to_string)+ .collect::<Vec<_>>();+ if targets.iter().any(String::is_empty) {+ return Err(format!("route {model} must contain non-empty targets"));+ }+ if targets.iter().collect::<BTreeSet<_>>().len() != targets.len() {+ return Err(format!("route {model} must contain unique targets"));+ }+ Ok(Self {+ model: model.to_string(),+ targets,+ })+ }+}++#[derive(Clone, Copy, Debug, ValueEnum)]+enum UpstreamFormat {+ #[value(name = "openai-chat")]+ OpenAiChat,+ #[value(name = "openai-responses")]+ OpenAiResponses,+ #[value(name = "anthropic")]+ Anthropic,+}++impl UpstreamFormat {+ fn backend(self, config: HttpBackendConfig) -> Backend {+ match self {+ Self::OpenAiChat => Backend::OpenAiChat(config),+ Self::OpenAiResponses => Backend::OpenAiResponses(config),+ Self::Anthropic => Backend::Anthropic(config),+ }+ }+}+/// Command-line arguments accepted by the Rust server binary.#[derive(Debug, Parser)]#[command(name = "switchyard-server",- about = "Run the Rust Switchyard server from a components-v2 profile config",+ about = "Run uniform random routing with libsy",version)]pub(crate) struct ServerArgs {- /// Path to a components-v2 profile config file.- #[arg(short, long, env = "SWITCHYARD_PROFILE_CONFIG", value_name = "PATH")]- pub(crate) config: PathBuf,+ /// Random route as MODEL=TARGET[,TARGET...]. Repeat to serve multiple routes.+ #[arg(+ long = "route",+ required = true,+ value_name = "MODEL=TARGET[,TARGET...]"+ )]+ routes: Vec<RandomRouteSpec>,++ /// Base URL shared by the configured upstream targets.+ #[arg(long, env = "SWITCHYARD_UPSTREAM_BASE_URL")]+ base_url: String,++ /// Upstream API key. Omit when the backend needs no authentication.+ #[arg(long, env = "SWITCHYARD_UPSTREAM_API_KEY")]+ api_key: Option<String>,++ /// Provider wire format used by the upstream backend.+ #[arg(long, value_enum, default_value = "openai-chat")]+ upstream_format: UpstreamFormat,/// Host address to bind.#[arg(long, default_value_t = DEFAULT_HOST)]- pub(crate) host: IpAddr,+ host: IpAddr,/// Port to bind.#[arg(short, long, default_value_t = DEFAULT_PORT)]- pub(crate) port: u16,+ port: u16,/// TCP listen backlog passed to the socket before Axum accepts traffic.#[arg(long, default_value_t = DEFAULT_LISTEN_BACKLOG)]- pub(crate) backlog: u32,+ backlog: u32,- /// Validate and build the config without starting the HTTP listener.+ /// Validate the algorithm and client configuration without binding a socket.#[arg(long)]- pub(crate) dry_run: bool,+ dry_run: bool,- /// TLS certificate path, PEM format+ /// TLS certificate path in PEM format.#[arg(long, requires = "tls_key")]- pub(crate) tls_cert: Option<PathBuf>,+ tls_cert: Option<PathBuf>,- /// TLS certificate key path, PEM format+ /// TLS private-key path in PEM format.#[arg(long, requires = "tls_cert")]- pub(crate) tls_key: Option<PathBuf>,+ tls_key: Option<PathBuf>,}impl ServerArgs {@@ -56,30 +135,108 @@ impl ServerArgs {Self::parse()}- fn into_options(self) -> Result<ServerRunOptions> {- let mut tls_options = None;- if let (Some(cert), Some(key)) = (self.tls_cert, self.tls_key) {- if !cert.exists() || !key.exists() {- return Err(SwitchyardError::InvalidConfig(format!(- "Invalid path in --tls-cert {} or --tls-key {}. File does not exist.",- cert.display(),- key.display()- )));+ fn into_runtime(self) -> ServerResult<(ServerState, ServerRunOptions)> {+ if self.base_url.trim().is_empty() {+ return Err(ServerError::new("--base-url must not be empty"));+ }++ let backend = self.upstream_format.backend(HttpBackendConfig {+ base_url: self.base_url,+ api_key: self.api_key,+ extra_headers: BTreeMap::new(),+ });+ let target_models = self+ .routes+ .iter()+ .flat_map(|route| route.targets.iter())+ .collect::<BTreeSet<_>>();+ let model_configs = target_models+ .into_iter()+ .map(|model| ModelConfig::new(model.as_str(), backend.clone(), None))+ .collect::<Vec<_>>();+ let client: Arc<dyn RoutedLlmClient> = Arc::new(+ TranslatingLlmClient::new(&model_configs)+ .map_err(|error| ServerError::new(error.to_string()))?,+ );+ let routes = self.routes.into_iter().map(|route| {+ let target_set = LlmTargetSet::new(+ route+ .targets+ .iter()+ .map(|model| LlmTarget {+ semantic_name: model.clone(),+ llm_client: Some(Arc::clone(&client)),+ })+ .collect(),+ );+ let algorithm: Arc<dyn Algorithm> = Arc::new(Random::new(target_set));+ (route.model, algorithm)+ });+ let state = ServerState::new(routes)?;++ let tls = match (self.tls_cert, self.tls_key) {+ (Some(cert), Some(key)) => {+ if !cert.exists() || !key.exists() {+ return Err(ServerError::new(format!(+ "invalid --tls-cert {} or --tls-key {}: file does not exist",+ cert.display(),+ key.display()+ )));+ }+ Some(TlsOptions { cert, key })}- tls_options = Some(TLSOptions { cert, key })+ _ => None,};- Ok(ServerRunOptions {- config: self.config,+ let options = ServerRunOptions {addr: SocketAddr::new(self.host, self.port),backlog: self.backlog,dry_run: self.dry_run,- tls: tls_options,- })+ tls,+ };+ Ok((state, options))}}-/// Loads config, optionally validates it, then starts the Rust server.-pub(crate) async fn run(args: ServerArgs) -> Result<()> {- let opts = args.into_options()?;- run_server(opts).await+/// Builds the random algorithm and starts the server.+pub(crate) async fn run(args: ServerArgs) -> ServerResult<()> {+ let (state, options) = args.into_runtime()?;+ run_server(state, options).await+}++#[cfg(test)]+mod tests {+ use super::*;++ fn runtime(routes: &[&str]) -> ServerResult<ServerState> {+ let mut args = vec!["switchyard-server", "--base-url", "http://127.0.0.1:9/v1"];+ for route in routes {+ args.extend(["--route", route]);+ }+ let args = ServerArgs::try_parse_from(args)+ .map_err(|error| ServerError::new(error.to_string()))?;+ args.into_runtime().map(|(state, _)| state)+ }++ #[test]+ fn routes_build_independent_algorithms() -> ServerResult<()> {+ let state = runtime(&[+ "switchyard/general=model/a,model/b",+ "switchyard/coding=model/c,model/d",+ ])?;+ assert_eq!(+ state.models().collect::<Vec<_>>(),+ ["switchyard/coding", "switchyard/general"]+ );+ Ok(())+ }++ #[test]+ fn invalid_routes_are_rejected() {+ for routes in [+ &["switchyard/general=model/a,model/a"][..],+ &["switchyard/general=model/a", "switchyard/general=model/b"],+ ] {+ assert!(runtime(routes).is_err());+ }+ }}diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex 50595aaab..a49e4e354 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -1,19 +1,16 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-//! Rust HTTP server surface for components-v2 profile configs.-//!-//! The serving path is profile-native: config files load into-//! `ProfileConfigPlan`, plans build `Profile` runtimes, and HTTP requests-//! call `Profile::run()` directly.+//! Rust HTTP server for libsy algorithms.-mod registry;mod response;mod sse;use std::collections::BTreeMap;+use std::error::Error;+use std::fmt::{Display, Formatter};use std::net::SocketAddr;-use std::path::{Path, PathBuf};+use std::path::PathBuf;use std::sync::Arc;use std::time::Duration;@@ -23,88 +20,108 @@ use axum::response::{IntoResponse, Response};use axum::routing::{get, post};use axum::{Json, Router};use axum_server::tls_rustls::RustlsConfig;+use libsy::{Algorithm, Context, Decision, Metadata, Request};use serde_json::{json, Value};+use switchyard_llm_client::LlmClientError;use tokio::net::{TcpListener, TcpSocket};-use switchyard_components_v2::{- parse_profile_config_path, profile_stats_accumulator, ProfileConfigPlan, ProfileInput,- ProfileResponse, RequestMetadata, RoutingMetadata,-};-use switchyard_core::{ChatRequest, ChatRequestType, RequestId, Result, SwitchyardError};-use switchyard_translation::{TranslationEngine, TranslationPolicy, WireFormat};+use switchyard_translation::{decode_request, WireFormat};-pub use registry::{ProfileRegistry, ServedModel};--use crate::response::{translate_chain_response, TranslatedResponse};+use crate::response::into_http_response;/// Default TCP listen backlog used by the Rust server.pub const DEFAULT_LISTEN_BACKLOG: u32 = 65_535;const HEADER_SELECTED_MODEL: &str = "x-model-router-selected-model";-const HEADER_SELECTED_TIER: &str = "x-model-router-selected-tier";-const HEADER_CONFIDENCE: &str = "x-model-router-confidence";-const HEADER_ROUTER_VERSION: &str = "x-model-router-version";-const HEADER_TOLERANCE: &str = "x-model-router-tolerance";const HEADER_RATIONALE: &str = "x-model-router-rationale";const MAX_ROUTING_HEADER_VALUE_LEN: usize = 512;+type BoxError = Box<dyn Error + Send + Sync>;++/// Error returned while configuring or running the server.+#[derive(Debug)]+pub struct ServerError {+ message: String,+}++impl ServerError {+ /// Creates a server error with a user-facing message.+ pub fn new(message: impl Into<String>) -> Self {+ Self {+ message: message.into(),+ }+ }+}++impl Display for ServerError {+ fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {+ formatter.write_str(&self.message)+ }+}++impl Error for ServerError {}++/// Result returned by server setup and lifecycle operations.+pub type ServerResult<T> = std::result::Result<T, ServerError>;+/// Shared server state used by all endpoint handlers.#[derive(Clone)]pub struct ServerState {- registry: Arc<ProfileRegistry>,- translation: Arc<TranslationEngine>,- translation_policy: TranslationPolicy,+ routes: Arc<BTreeMap<String, Arc<dyn Algorithm>>>,}impl ServerState {- /// Creates server state for a profile registry.- pub fn new(registry: ProfileRegistry) -> Self {- Self {- registry: Arc::new(registry),- translation: Arc::new(TranslationEngine::default()),- translation_policy: TranslationPolicy::default(),+ /// Creates server state from route model IDs and their libsy algorithms.+ pub fn new(+ routes: impl IntoIterator<Item = (String, Arc<dyn Algorithm>)>,+ ) -> ServerResult<Self> {+ let mut entries = BTreeMap::new();+ for (model, algorithm) in routes {+ let model = model.trim();+ if model.is_empty() {+ return Err(ServerError::new("route model must not be empty"));+ }+ if entries.insert(model.to_string(), algorithm).is_some() {+ return Err(ServerError::new(format!("duplicate route model {model}")));+ }}+ if entries.is_empty() {+ return Err(ServerError::new("at least one algorithm route is required"));+ }+ Ok(Self {+ routes: Arc::new(entries),+ })}- /// Builds server state from a resolved profile config plan.- pub fn from_plan(plan: &ProfileConfigPlan) -> Result<Self> {- Ok(Self::new(ProfileRegistry::from_plan(plan)?))- }-- /// Returns the profile registry used by this server.- pub fn registry(&self) -> &ProfileRegistry {- self.registry.as_ref()+ /// Returns the route model IDs served by the configured algorithms.+ pub fn models(&self) -> impl Iterator<Item = &str> {+ self.routes.keys().map(String::as_str)}- /// Dispatches one request to the profile selected by its `model` field.- async fn run_profile(&self, input: ProfileInput) -> Result<ProfileResponse> {- let profile = self.registry.lookup(input.request.model())?;- profile.run(input).await+ fn algorithm_for_model(&self, model: &str) -> Option<Arc<dyn Algorithm>> {+ self.routes.get(model).map(Arc::clone)}}-/// Runtime options shared by the Rust binary and Python binding.+/// Runtime options shared by server entry points.#[derive(Clone, Debug)]pub struct ServerRunOptions {- /// Path to the components-v2 profile config file.- pub config: PathBuf,/// Socket address to bind.pub addr: SocketAddr,/// TCP listen backlog.pub backlog: u32,- /// Validate and print public model IDs without binding a socket.+ /// Validate runtime construction without binding a socket.pub dry_run: bool,- /// If this is set server is HTTPS.- /// Caller must validate that the files exist.- pub tls: Option<TLSOptions>,+ /// TLS certificate configuration, when HTTPS is enabled.+ pub tls: Option<TlsOptions>,}+/// TLS certificate paths used by the server.#[derive(Clone, Debug)]-pub struct TLSOptions {- /// TLS certificate path, PEM format+pub struct TlsOptions {+ /// TLS certificate path in PEM format.pub cert: PathBuf,-- /// TLS certificate key path, PEM format+ /// TLS private-key path in PEM format.pub key: PathBuf,}@@ -114,19 +131,10 @@ impl ServerRunOptions {}}-/// Builds a server state by loading and resolving a profile config path.-fn state_from_config_path(path: impl AsRef<Path>) -> Result<ServerState> {- let document = parse_profile_config_path(path)?;- let plan = document.resolve()?;- ServerState::from_plan(&plan)-}--/// Entry point to this module.-/// Loads config, optionally validates it, then starts the Rust server.-pub async fn run_server(options: ServerRunOptions) -> Result<()> {- let state = state_from_config_path(&options.config)?;+/// Validates the runtime and starts the HTTP server unless `dry_run` is set.+pub async fn run_server(state: ServerState, options: ServerRunOptions) -> ServerResult<()> {if options.dry_run {- println!("{}", dry_run_summary(&options.config, state.registry()));+ println!("{}", dry_run_summary(&state));return Ok(());}@@ -136,7 +144,7 @@ pub async fn run_server(options: ServerRunOptions) -> Result<()> {addr: bound_addr,..options};- eprintln!("{}", startup_banner(&server_options, state.registry()));+ eprintln!("{}", startup_banner(&server_options, &state));let router = build_switchyard_router(state);if let Some(tls) = server_options.tls {serve_tls(listener, router, tls).await@@ -145,11 +153,9 @@ pub async fn run_server(options: ServerRunOptions) -> Result<()> {}}-async fn serve_tls(listener: TcpListener, router: Router, tls: TLSOptions) -> Result<()> {- // aws_lc_rs is the default but other crates pull in `ring` also,- // so rustls doesn't know which one to use. Tell it.- if let Err(e) = rustls::crypto::aws_lc_rs::default_provider().install_default() {- eprintln!("TLS crypto provider already installed: {e:?}");+async fn serve_tls(listener: TcpListener, router: Router, tls: TlsOptions) -> ServerResult<()> {+ if let Err(error) = rustls::crypto::aws_lc_rs::default_provider().install_default() {+ tracing::debug!(?error, "TLS crypto provider was already installed");}let config = RustlsConfig::from_pem_file(tls.cert, tls.key)@@ -160,46 +166,38 @@ async fn serve_tls(listener: TcpListener, router: Router, tls: TLSOptions) -> Relet shutdown_handle = handle.clone();tokio::spawn(async move {shutdown_signal().await;- // Drain connections. Later should loop on connection_count() > 0.shutdown_handle.graceful_shutdown(Some(Duration::from_secs(2)));});let std_listener = listener.into_std().map_err(server_io_error)?;axum_server::from_tcp_rustls(std_listener, config).map_err(server_io_error)?- .handle(handle.clone())+ .handle(handle).serve(router.into_make_service()).await.map_err(server_io_error)}-/// Serves a Switchyard router on an already-bound TCP listener.-async fn serve(listener: TcpListener, router: Router) -> Result<()> {+async fn serve(listener: TcpListener, router: Router) -> ServerResult<()> {axum::serve(listener, router).with_graceful_shutdown(shutdown_signal()).await.map_err(server_io_error)}-/// Builds an Axum router with the same primary endpoint paths as the Python app.-/// Public so that integration tests can see it.+/// Builds an Axum router for the supported LLM wire formats.pub fn build_switchyard_router(state: ServerState) -> Router {Router::new().route("/v1/chat/completions", post(openai_chat_completions)).route("/v1/messages", post(anthropic_messages)).route("/v1/responses", post(openai_responses)).route("/v1/models", get(models))- // Keep the legacy routing stats aliases wired to the same handlers.- .route("/v1/stats", get(stats))- .route("/v1/stats/reset", post(reset_stats))- .route("/v1/routing/stats", get(stats))- .route("/v1/routing/stats/reset", post(reset_stats)).route("/health", get(health)).fallback(not_found).with_state(state)}-fn bind_tcp_listener(addr: SocketAddr, backlog: u32) -> Result<TcpListener> {+fn bind_tcp_listener(addr: SocketAddr, backlog: u32) -> ServerResult<TcpListener> {let socket = if addr.is_ipv4() {TcpSocket::new_v4()} else {@@ -212,8 +210,8 @@ fn bind_tcp_listener(addr: SocketAddr, backlog: u32) -> Result<TcpListener> {socket.listen(backlog).map_err(server_io_error)}-fn server_io_error(error: std::io::Error) -> SwitchyardError {- SwitchyardError::Other(error.to_string())+fn server_io_error(error: std::io::Error) -> ServerError {+ ServerError::new(error.to_string())}async fn shutdown_signal() {@@ -231,17 +229,7 @@ async fn openai_chat_completions(headers: HeaderMap,body: std::result::Result<Json<Value>, JsonRejection>,) -> Response {- let body = match llm_json_body(body) {- Ok(body) => body,- Err(response) => return *response,- };- handle_llm_request(- state,- ChatRequest::openai_chat(body),- WireFormat::OpenAiChat,- metadata_from_headers(&headers, ChatRequestType::OpenAiChat),- )- .await+ handle_endpoint(state, headers, body, WireFormat::OpenAiChat).await}async fn anthropic_messages(@@ -249,116 +237,127 @@ async fn anthropic_messages(headers: HeaderMap,body: std::result::Result<Json<Value>, JsonRejection>,) -> Response {- let body = match llm_json_body(body) {- Ok(body) => body,- Err(response) => return *response,- };- handle_llm_request(- state,- ChatRequest::anthropic(body),- WireFormat::AnthropicMessages,- metadata_from_headers(&headers, ChatRequestType::Anthropic),- )- .await+ handle_endpoint(state, headers, body, WireFormat::AnthropicMessages).await}async fn openai_responses(State(state): State<ServerState>,headers: HeaderMap,body: std::result::Result<Json<Value>, JsonRejection>,+) -> Response {+ handle_endpoint(state, headers, body, WireFormat::OpenAiResponses).await+}++async fn handle_endpoint(+ state: ServerState,+ headers: HeaderMap,+ body: std::result::Result<Json<Value>, JsonRejection>,+ wire_format: WireFormat,) -> Response {let body = match llm_json_body(body) {Ok(body) => body,- Err(response) => return *response,+ Err(message) => return invalid_body_error(message),};- handle_llm_request(- state,- ChatRequest::openai_responses(body),- WireFormat::OpenAiResponses,- metadata_from_headers(&headers, ChatRequestType::OpenAiResponses),- )- .await+ handle_llm_request(state, headers, body, wire_format).await}fn llm_json_body(body: std::result::Result<Json<Value>, JsonRejection>,-) -> std::result::Result<Value, Box<Response>> {+) -> std::result::Result<Value, String> {match body {Ok(Json(value)) if value.is_object() => Ok(value),- Ok(_) => Err(Box::new(invalid_body_error(- "Request body must be a JSON object",- ))),- Err(error) => Err(Box::new(invalid_body_error(format!(- "Request body must be valid JSON: {error}"- )))),+ Ok(_) => Err("Request body must be a JSON object".to_string()),+ Err(error) => Err(format!("Request body must be valid JSON: {error}")),}}async fn handle_llm_request(state: ServerState,- request: ChatRequest,- target_format: WireFormat,- metadata: RequestMetadata,+ headers: HeaderMap,+ body: Value,+ wire_format: WireFormat,) -> Response {- if let Err(error) = request.validate() {- return llm_error(error);- }+ let llm_request = match decode_request(wire_format, &body) {+ Ok(request) => request,+ Err(error) => return invalid_body_error(error.to_string()),+ };+ let Some(requested_model) = llm_request+ .model+ .clone()+ .filter(|model| !model.trim().is_empty())+ else {+ return error_response(+ StatusCode::BAD_REQUEST,+ "request body must include a non-empty string `model`",+ "invalid_request_error",+ "invalid_request_error",+ );+ };+ let Some(algorithm) = state.algorithm_for_model(&requested_model) else {+ return error_response(+ StatusCode::NOT_FOUND,+ format!("No route registered for model {requested_model}"),+ "model_not_found",+ "model_not_found",+ );+ };- let profile_response = match state.run_profile(ProfileInput { request, metadata }).await {- Ok(response) => response,- Err(error) => return llm_error(error),+ let request = Request {+ llm_request,+ raw_request: Some(body),+ metadata: Some(metadata_from_headers(&headers)),};- let (response, routing_metadata) = profile_response.into_parts();-- let mut response = match translate_chain_response(- response,- target_format,- Arc::clone(&state.translation),- state.translation_policy.clone(),- ) {- Ok(TranslatedResponse::Buffered(body)) => Json(body).into_response(),- Ok(TranslatedResponse::Stream(stream)) => stream.into_response(),- Err(error) => return server_error(error.to_string()),+ let (trace, response) = match algorithm.run(Context::default(), request).await {+ Ok(result) => result,+ Err(error) => return algorithm_error(error),};- attach_routing_metadata_headers(&mut response, routing_metadata.as_ref());- response-}-fn attach_routing_metadata_headers(response: &mut Response, metadata: Option<&RoutingMetadata>) {- let Some(metadata) = metadata else {- return;+ let mut response = match into_http_response(response, wire_format, Some(requested_model)) {+ Ok(response) => response,+ Err(error) => return server_error(error.to_string()),};- for (name, value) in routing_metadata_headers(metadata) {- let name = HeaderName::from_static(name);- let Ok(value) = HeaderValue::from_str(&value) else {- continue;- };- response.headers_mut().insert(name, value);+ if let Some(decision) = trace.last() {+ attach_routing_headers(&mut response, decision.as_ref());}+ response}-fn routing_metadata_headers(metadata: &RoutingMetadata) -> Vec<(&'static str, String)> {- [- (HEADER_SELECTED_MODEL, text_header(&metadata.selected_model)),- (HEADER_SELECTED_TIER, text_header(&metadata.selected_tier)),- (HEADER_CONFIDENCE, number_header(metadata.confidence)),- (HEADER_ROUTER_VERSION, text_header(&metadata.router_version)),- (HEADER_TOLERANCE, number_header(metadata.tolerance)),- (HEADER_RATIONALE, text_header(&metadata.rationale)),- ]- .into_iter()- .filter_map(|(name, value)| value.map(|value| (name, value)))- .collect()+fn metadata_from_headers(headers: &HeaderMap) -> Metadata {+ let headers = normalized_headers(headers);+ let mut metadata = Metadata::from_headers(&headers);+ metadata.http_headers = Some(headers);+ metadata}-fn text_header(value: &Option<String>) -> Option<String> {- value.as_deref().and_then(sanitize_routing_header_value)+fn normalized_headers(headers: &HeaderMap) -> BTreeMap<String, String> {+ headers+ .iter()+ .filter_map(|(name, value)| {+ value+ .to_str()+ .ok()+ .map(|value| (name.as_str().to_ascii_lowercase(), value.to_string()))+ })+ .collect()+}++fn attach_routing_headers(response: &mut Response, decision: &dyn Decision) {+ insert_routing_header(response, HEADER_SELECTED_MODEL, decision.selected_model());+ if let Some(reasoning) = decision.reasoning() {+ insert_routing_header(response, HEADER_RATIONALE, reasoning);+ }}-fn number_header(value: Option<f64>) -> Option<String> {- value- .filter(|value| value.is_finite())- .map(|value| value.to_string())+fn insert_routing_header(response: &mut Response, name: &'static str, value: &str) {+ let Some(value) = sanitize_routing_header_value(value) else {+ return;+ };+ let Ok(value) = HeaderValue::from_str(&value) else {+ return;+ };+ response+ .headers_mut()+ .insert(HeaderName::from_static(name), value);}fn sanitize_routing_header_value(value: &str) -> Option<String> {@@ -366,112 +365,83 @@ fn sanitize_routing_header_value(value: &str) -> Option<String> {(!value.is_empty()).then(|| value.chars().take(MAX_ROUTING_HEADER_VALUE_LEN).collect())}-fn metadata_from_headers(headers: &HeaderMap, inbound_format: ChatRequestType) -> RequestMetadata {- RequestMetadata {- request_id: request_id_from_headers(headers),- inbound_format: Some(inbound_format),- headers: normalized_headers(headers),- }-}--fn request_id_from_headers(headers: &HeaderMap) -> Option<RequestId> {- headers- .get("x-request-id")- .and_then(|value| value.to_str().ok())- .and_then(|value| RequestId::new(value.to_string()).ok())-}--fn normalized_headers(headers: &HeaderMap) -> BTreeMap<String, Vec<String>> {- let mut normalized = BTreeMap::<String, Vec<String>>::new();- for (name, value) in headers {- let Ok(value) = value.to_str() else {- continue;- };- normalized- .entry(name.as_str().to_ascii_lowercase())- .or_default()- .push(value.to_string());- }- normalized-}--fn llm_error(error: SwitchyardError) -> Response {+fn algorithm_error(error: BoxError) -> Response {+ let Some(error) = error.downcast_ref::<LlmClientError>() else {+ return server_error(error.to_string());+ };match error {- SwitchyardError::ModelNotFound { model } => (- StatusCode::NOT_FOUND,- Json(json!({- "error": {- "message": format!("No route registered for model {}", model.as_str()),- "type": "model_not_found",- "code": "model_not_found",- }- })),- )- .into_response(),- SwitchyardError::InvalidConfig(message) | SwitchyardError::InvalidRequest(message) => (+ LlmClientError::UnknownModel(model) => error_response(+ StatusCode::BAD_GATEWAY,+ format!("No upstream backend configured for model {model}"),+ "upstream_error",+ "upstream_model_not_found",+ ),+ LlmClientError::UnknownModelFormat { model, format } => error_response(+ StatusCode::BAD_GATEWAY,+ format!("No upstream backend configured for model {model} and format {format}"),+ "upstream_error",+ "upstream_format_not_found",+ ),+ LlmClientError::MissingModel => error_response(StatusCode::BAD_REQUEST,- Json(json!({- "error": {- "message": message,- "type": "invalid_request_error",- "code": "invalid_request_error",- }- })),- )- .into_response(),- SwitchyardError::InvalidId(error) => (+ error.to_string(),+ "invalid_request_error",+ "invalid_request_error",+ ),+ LlmClientError::ContextWindowExceeded { message, .. } => error_response(StatusCode::BAD_REQUEST,- Json(json!({- "error": {- "message": error.to_string(),- "type": "invalid_request_error",- "code": "invalid_request_error",- }- })),- )- .into_response(),- SwitchyardError::UpstreamHttp {- provider,- status_code,+ message,+ "invalid_request_error",+ "context_length_exceeded",+ ),+ LlmClientError::UpstreamHttp { status, body } => error_response(+ StatusCode::from_u16(*status).unwrap_or(StatusCode::BAD_GATEWAY),body,- } => (- StatusCode::from_u16(status_code).unwrap_or(StatusCode::BAD_GATEWAY),- Json(json!({- "error": {- "message": body,- "type": "upstream_error",- "code": "upstream_error",- "provider": provider,- }- })),- )- .into_response(),- error => server_error(error.to_string()),+ "upstream_error",+ "upstream_error",+ ),+ LlmClientError::Translation(message)+ | LlmClientError::Transport(message)+ | LlmClientError::Stream(message) => error_response(+ StatusCode::BAD_GATEWAY,+ message,+ "upstream_error",+ "upstream_error",+ ),}}-fn server_error(message: String) -> Response {- (+fn server_error(message: impl Into<String>) -> Response {+ error_response(StatusCode::INTERNAL_SERVER_ERROR,- Json(json!({- "error": {- "message": message,- "type": "server_error",- "code": "server_error",- }- })),+ message,+ "server_error",+ "server_error",)- .into_response()}fn invalid_body_error(message: impl Into<String>) -> Response {- (+ error_response(StatusCode::BAD_REQUEST,+ message,+ "invalid_request_error",+ "invalid_body",+ )+}++fn error_response(+ status: StatusCode,+ message: impl Into<String>,+ error_type: &'static str,+ code: &'static str,+) -> Response {+ (+ status,Json(json!({"error": {"message": message.into(),- "type": "invalid_request_error",- "code": "invalid_body",+ "type": error_type,+ "code": code,}})),)@@ -479,22 +449,7 @@ fn invalid_body_error(message: impl Into<String>) -> Response {}async fn models(State(state): State<ServerState>) -> Json<Value> {- let entries = state.registry().served_models();- Json(model_list_payload(&entries))-}--async fn stats() -> Response {- match profile_stats_accumulator().snapshot() {- Ok(snapshot) => Json(json!(snapshot)).into_response(),- Err(error) => server_error(error.to_string()),- }-}--async fn reset_stats() -> Response {- match profile_stats_accumulator().reset() {- Ok(()) => Json(json!({"status": "reset"})).into_response(),- Err(error) => server_error(error.to_string()),- }+ Json(model_list_payload(state.models()))}async fn health() -> Json<Value> {@@ -502,27 +457,21 @@ async fn health() -> Json<Value> {}async fn not_found() -> Response {- (+ error_response(StatusCode::NOT_FOUND,- Json(json!({- "detail": "Not Found",- })),+ "Not Found",+ "not_found",+ "endpoint_not_found",)- .into_response()}-fn model_list_payload(entries: &[ServedModel]) -> Value {- let data = entries.iter().map(model_entry_json).collect::<Vec<_>>();- let model_ids = entries- .iter()- .map(|entry| entry.id.as_str().to_string())- .collect::<Vec<_>>();+fn model_list_payload<'a>(models: impl IntoIterator<Item = &'a str>) -> Value {+ let model_ids = models.into_iter().map(str::to_string).collect::<Vec<_>>();let first_id = model_ids.first().cloned();let last_id = model_ids.last().cloned();-json!({"object": "list",- "data": data,+ "data": model_ids.iter().map(|model| model_entry_json(model)).collect::<Vec<_>>(),"first_id": first_id,"last_id": last_id,"has_more": false,@@ -531,14 +480,14 @@ fn model_list_payload(entries: &[ServedModel]) -> Value {})}-fn model_entry_json(entry: &ServedModel) -> Value {+fn model_entry_json(model: &str) -> Value {json!({- "id": entry.id.as_str(),+ "id": model,"object": "model","type": "model","created": 0,"owned_by": "switchyard",- "display_name": entry.display_name,+ "display_name": model,"capabilities": {"streaming": true,"tool_calling": null,@@ -552,115 +501,26 @@ fn model_entry_json(entry: &ServedModel) -> Value {})}-fn startup_banner(options: &ServerRunOptions, registry: &ProfileRegistry) -> String {- let entries = registry.served_models();+fn startup_banner(options: &ServerRunOptions, state: &ServerState) -> String {let scheme = if options.is_tls() { "https" } else { "http" };- let listen_url = url_for_addr(scheme, options.addr);- let local_url = local_url_for_addr(scheme, options.addr);- let mut output = String::new();-- push_line(&mut output, "Switchyard Rust profile server");- push_line(- &mut output,- format!(" config: {}", options.config.display()),- );- push_line(&mut output, format!(" listening: {listen_url}"));- if local_url != listen_url {- push_line(&mut output, format!(" local: {local_url}"));- }- push_line(&mut output, "");- push_line(&mut output, " endpoints:");- push_line(&mut output, " GET /health");- push_line(&mut output, " GET /v1/models");- push_line(&mut output, " POST /v1/chat/completions");- push_line(&mut output, " POST /v1/messages");- push_line(&mut output, " POST /v1/responses");- push_line(&mut output, " GET /v1/routing/stats");- push_line(&mut output, " POST /v1/routing/stats/reset");- push_line(&mut output, "");- push_line(&mut output, " available models:");- for entry in &entries {- if entry.display_name == entry.id.as_str() {- push_line(&mut output, format!(" - {}", entry.id.as_str()));- } else {- push_line(- &mut output,- format!(" - {} ({})", entry.id.as_str(), entry.display_name),- );- }- }- push_line(&mut output, "");- push_line(&mut output, " try:");- push_line(&mut output, format!(" curl -s {local_url}/health"));- push_line(&mut output, format!(" curl -s {local_url}/v1/models"));- if let Some(entry) = entries.first() {- let chat_payload = json!({- "model": entry.id.as_str(),- "messages": [{"role": "user", "content": "Say OK"}],- "max_tokens": 256, // Space for thinking tokens- });- push_line(- &mut output,- format!(" curl -s {local_url}/v1/chat/completions -H 'content-type: application/json' -d '{chat_payload}'")- );- push_line(- &mut output,- format!(" curl -s {local_url}/v1/messages -H 'content-type: application/json' -H 'anthropic-version: 2023-06-01' -d '{chat_payload}'")- );-- let responses_payload = json!({- "model": entry.id.as_str(),- "max_output_tokens": 256,- "input": "Say OK",- });- push_line(&mut output, format!(" curl -s {local_url}/v1/responses -H 'content-type: application/json' -d '{responses_payload}'"));- }- if options.is_tls() {- push_line(- &mut output,- " For self-signed certificates use `curl --insecure ..`",- );- }- push_line(&mut output, "");- push_line(&mut output, " stop: Ctrl-C");- output-}--fn dry_run_summary(path: &Path, registry: &ProfileRegistry) -> String {- let entries = registry.served_models();- let mut output = String::new();- push_line(- &mut output,- format!(- "config OK: {}, public_models={}",- path.display(),- entries.len()- ),- );- for entry in &entries {- push_line(&mut output, format!(" - {}", entry.id.as_str()));- }- output+ format!(+ "Switchyard libsy server\n listening: {}\n routes: {}",+ url_for_addr(scheme, options.addr),+ state.models().collect::<Vec<_>>().join(", ")+ )}-fn push_line(output: &mut String, line: impl AsRef<str>) {- output.push_str(line.as_ref());- output.push('\n');+fn dry_run_summary(state: &ServerState) -> String {+ format!(+ "server OK: {}",+ state.models().collect::<Vec<_>>().join(", ")+ )}fn url_for_addr(scheme: &'static str, addr: SocketAddr) -> String {format!("{scheme}://{}:{}", host_for_url(addr.ip()), addr.port())}-fn local_url_for_addr(scheme: &'static str, addr: SocketAddr) -> String {- let host = match addr.ip() {- std::net::IpAddr::V4(ip) if ip.is_unspecified() => "127.0.0.1".to_string(),- std::net::IpAddr::V6(ip) if ip.is_unspecified() => "[::1]".to_string(),- ip => host_for_url(ip),- };- format!("{scheme}://{host}:{}", addr.port())-}-fn host_for_url(ip: std::net::IpAddr) -> String {match ip {std::net::IpAddr::V4(ip) => ip.to_string(),diff --git a/crates/switchyard-server/src/registry.rs b/crates/switchyard-server/src/registry.rsdeleted file mode 100644index 20b829c05..000000000--- a/crates/switchyard-server/src/registry.rs+++ /dev/null@@ -1,119 +0,0 @@-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-// SPDX-License-Identifier: Apache-2.0--//! Profile registry used by the components-v2 Rust server.--use std::sync::Arc;--use serde::{Deserialize, Serialize};-use switchyard_components_v2::{- PassthroughProfileConfig, Profile, ProfileConfig, ProfileConfigPlan,-};-use switchyard_core::{ModelId, Result, SwitchyardError};--/// Public model entry advertised by `/v1/models`.-#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]-pub struct ServedModel {- /// Public model or route ID accepted in inbound request bodies.- pub id: ModelId,- /// Human-readable label shown by CLI startup logs and `/v1/models`.- pub display_name: String,-}--#[derive(Clone)]-struct RegistryEntry {- model: ServedModel,- profile: Arc<dyn Profile>,-}--/// Exact-match registry from inbound `model` values to components-v2 profiles.-#[derive(Clone, Default)]-pub struct ProfileRegistry {- entries: Vec<RegistryEntry>,-}--impl ProfileRegistry {- /// Builds a registry from already-built exact-match profile runtimes.- pub fn from_profiles(- entries: impl IntoIterator<Item = (ModelId, Arc<dyn Profile>, String)>,- ) -> Result<Self> {- let mut registry = Self::default();- for (model_id, profile, display_name) in entries {- registry.insert(model_id, profile, display_name)?;- }- Ok(registry)- }-- /// Builds a registry from a resolved profile config plan.- pub fn from_plan(plan: &ProfileConfigPlan) -> Result<Self> {- let mut registry = Self::default();-- for profile_id in plan.profile_ids() {- let model_id = ModelId::new(profile_id.as_str())?;- let profile: Arc<dyn Profile> = Arc::from(plan.build_profile(profile_id)?);- let display_name = plan.profile_type(profile_id).unwrap_or("profile");- registry.insert(model_id, profile, display_name)?;- }-- for (_target_id, target) in plan.targets() {- let profile: Arc<dyn Profile> = Arc::from(- PassthroughProfileConfig {- target: target.clone(),- }- .build_boxed()?,- );- registry.insert(- ModelId::new(target.id.as_str())?,- Arc::clone(&profile),- target.model.as_str(),- )?;- }-- Ok(registry)- }-- /// Returns the profile for an inbound request model.- pub fn lookup(&self, model: Option<&str>) -> Result<Arc<dyn Profile>> {- let Some(model) = model else {- return Err(SwitchyardError::InvalidRequest(- "request body must include a non-empty string `model`".to_string(),- ));- };- let model = ModelId::new(model)?;- self.entries- .iter()- .find(|entry| entry.model.id == model)- .map(|entry| Arc::clone(&entry.profile))- .ok_or(SwitchyardError::ModelNotFound { model })- }-- /// Returns model entries in deterministic registration order.- pub fn served_models(&self) -> Vec<ServedModel> {- self.entries- .iter()- .map(|entry| entry.model.clone())- .collect()- }-- fn insert(- &mut self,- model_id: ModelId,- profile: Arc<dyn Profile>,- display_name: impl Into<String>,- ) -> Result<()> {- if self.entries.iter().any(|entry| entry.model.id == model_id) {- return Err(SwitchyardError::DuplicateRegistration {- kind: "model",- id: model_id.to_string(),- });- }- self.entries.push(RegistryEntry {- model: ServedModel {- id: model_id,- display_name: display_name.into(),- },- profile,- });- Ok(())- }-}diff --git a/crates/switchyard-server/src/response.rs b/crates/switchyard-server/src/response.rsindex 0a27718e3..9488d1764 100644--- a/crates/switchyard-server/src/response.rs+++ b/crates/switchyard-server/src/response.rs@@ -1,144 +1,36 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-//! Response translation glue for Rust server endpoints.+//! Response encoding glue for libsy server endpoints.-use std::sync::Arc;+use std::error::Error;-use axum::response::sse::Sse;-use serde_json::Value;-use switchyard_core::{BoxResponseStream, ChatResponse, Result, StreamEvent, SwitchyardError};-use switchyard_translation::{- StreamTranslationState, TranslationEngine, TranslationPolicy, WireFormat,-};+use axum::response::{IntoResponse, Response as HttpResponse};+use axum::Json;+use libsy::{LlmResponse, Response as AlgorithmResponse};+use switchyard_translation::{encode_aggregated_response, encode_stream, WireFormat};-use crate::sse::{frame_stream, SseFrameStream};+use crate::sse::frame_stream;-pub(crate) enum TranslatedResponse {- /// Complete JSON response body ready for an Axum JSON response.- Buffered(Value),- /// Framed SSE response ready for Axum streaming.- Stream(Sse<SseFrameStream>),-}+type BoxError = Box<dyn Error + Send + Sync>;-/// Translates a profile response into the endpoint's expected response shape.-pub(crate) fn translate_chain_response(- response: ChatResponse,+/// Encodes a libsy response into the endpoint's wire format.+pub(crate) fn into_http_response(+ response: AlgorithmResponse,target_format: WireFormat,- translation: Arc<TranslationEngine>,- policy: TranslationPolicy,-) -> Result<TranslatedResponse> {- match response {- ChatResponse::OpenAiCompletion(response) => translate_buffered_response(- response.into_body(),- WireFormat::OpenAiChat,- target_format,- translation,- &policy,- ),- ChatResponse::OpenAiResponsesCompletion(response) => translate_buffered_response(- response.into_body(),- WireFormat::OpenAiResponses,+ requested_model: Option<String>,+) -> Result<HttpResponse, BoxError> {+ match response.llm_response {+ LlmResponse::Agg(response) => Ok(Json(encode_aggregated_response(+ &response,target_format,- translation,- &policy,- ),- ChatResponse::AnthropicCompletion(response) => translate_buffered_response(- response.into_body(),- WireFormat::AnthropicMessages,+ requested_model.as_deref(),+ )?)+ .into_response()),+ LlmResponse::Stream(stream) => Ok(frame_stream(+ encode_stream(stream, target_format, requested_model)?,target_format,- translation,- &policy,- ),- ChatResponse::OpenAiStream(stream) => Ok(TranslatedResponse::Stream(- translate_stream_response(stream, WireFormat::OpenAiChat, target_format, translation),- )),- ChatResponse::OpenAiResponsesStream(stream) => {- Ok(TranslatedResponse::Stream(translate_stream_response(- stream,- WireFormat::OpenAiResponses,- target_format,- translation,- )))- }- ChatResponse::AnthropicStream(stream) => {- Ok(TranslatedResponse::Stream(translate_stream_response(- stream,- WireFormat::AnthropicMessages,- target_format,- translation,- )))- }- }-}--fn translate_buffered_response(- body: Value,- source_format: WireFormat,- target_format: WireFormat,- translation: Arc<TranslationEngine>,- policy: &TranslationPolicy,-) -> Result<TranslatedResponse> {- if source_format == target_format {- return Ok(TranslatedResponse::Buffered(body));+ )+ .into_response()),}-- let output = translation- .translate_response(source_format, target_format, &body, policy)- .map_err(|error| {- SwitchyardError::Other(format!(- "failed to translate {source_format} response to {target_format}: {error}"- ))- })?;- Ok(TranslatedResponse::Buffered(output.body))-}--fn translate_stream_response(- mut stream: BoxResponseStream,- source_format: WireFormat,- target_format: WireFormat,- translation: Arc<TranslationEngine>,-) -> Sse<SseFrameStream> {- let events = async_stream::try_stream! {- let mut translation_state = StreamTranslationState::new(source_format, target_format);- while let Some(event) = futures_util::StreamExt::next(&mut stream).await {- let event = event?;- match event {- StreamEvent::Json(value) => {- if source_format == target_format {- yield value;- } else {- for translated in translation.translate_event(- &mut translation_state,- source_format,- target_format,- &value,- ).map_err(|error| {- SwitchyardError::Other(format!(- "failed to translate {source_format} stream event to {target_format}: {error}"- ))- })? {- yield translated;- }- }- }- StreamEvent::Text(text) => {- yield Value::String(text);- }- }- }-- if source_format != target_format {- for translated in translation.finish_stream(&mut translation_state, target_format)- .map_err(|error| {- SwitchyardError::Other(format!(- "failed to finish {target_format} stream translation: {error}"- ))- })? {- yield translated;- }- }- };-- frame_stream(events, target_format)}diff --git a/crates/switchyard-server/src/sse.rs b/crates/switchyard-server/src/sse.rsindex 3d4f0e38d..d1c25ce4d 100644--- a/crates/switchyard-server/src/sse.rs+++ b/crates/switchyard-server/src/sse.rs@@ -3,35 +3,47 @@//! SSE framing helpers for OpenAI, Anthropic, and Responses endpoints.+use std::convert::Infallible;+use axum::response::sse::{Event, Sse};use futures_util::Stream;use serde_json::{json, Value};-use switchyard_core::{Result, SwitchyardError};-use switchyard_translation::WireFormat;+use switchyard_translation::{RawEventStream, WireFormat};/// Boxed stream type accepted by Axum's SSE response wrapper.pub(crate) type SseFrameStream =- std::pin::Pin<Box<dyn Stream<Item = std::result::Result<Event, SwitchyardError>> + Send>>;+ std::pin::Pin<Box<dyn Stream<Item = Result<Event, Infallible>> + Send>>;/// Converts translated JSON events into endpoint-specific SSE frames.pub(crate) fn frame_stream(- stream: impl Stream<Item = Result<Value>> + Send + 'static,+ stream: RawEventStream,target_format: WireFormat,) -> Sse<SseFrameStream> {let framed = async_stream::stream! {- let mut stream = Box::pin(stream);+ let mut stream = stream;+ let mut failed = false;while let Some(item) = futures_util::StreamExt::next(&mut stream).await {- match item {- Ok(value) => yield frame_event(target_format, value),+ let event = match item {+ Ok(value) => match frame_event(target_format, value) {+ Ok(event) => event,+ Err(error) => {+ failed = true;+ error_event(target_format, error.to_string())+ }+ },Err(error) => {tracing::warn!(error = %error, "stream iteration failed");- yield Ok(error_event(target_format, error.to_string()));- return;+ failed = true;+ error_event(target_format, error.to_string())}+ };+ yield Ok(event);+ if failed {+ break;}}- if target_format == WireFormat::OpenAiChat {+ if !failed && target_format == WireFormat::OpenAiChat {yield Ok(Event::default().data("[DONE]"));}};@@ -39,24 +51,16 @@ pub(crate) fn frame_stream(Sse::new(Box::pin(framed) as SseFrameStream)}-fn frame_event(- target_format: WireFormat,- value: Value,-) -> std::result::Result<Event, SwitchyardError> {+fn frame_event(target_format: WireFormat, value: Value) -> Result<Event, axum::Error> {match target_format {- WireFormat::OpenAiChat => Event::default()- .json_data(value)- .map_err(|error| SwitchyardError::Other(error.to_string())),+ WireFormat::OpenAiChat => Event::default().json_data(value),WireFormat::AnthropicMessages | WireFormat::OpenAiResponses => {let event_type = value.get("type").and_then(Value::as_str).unwrap_or("message").to_string();- Event::default()- .event(event_type)- .json_data(value)- .map_err(|error| SwitchyardError::Other(error.to_string()))+ Event::default().event(event_type).json_data(value)}}}@@ -86,3 +90,35 @@ fn error_event(target_format: WireFormat, message: String) -> Event {}}}++#[cfg(test)]+mod tests {+ use std::{error::Error, io};++ use axum::{body::to_bytes, response::IntoResponse};+ use futures_util::stream;++ use super::*;++ type TestResult = Result<(), Box<dyn Error + Send + Sync>>;++ #[tokio::test]+ async fn stream_error_terminates_without_done_marker() -> TestResult {+ let failure: Box<dyn Error + Send + Sync> = Box::new(io::Error::other("boom"));+ let stream: RawEventStream = Box::pin(stream::iter(vec![+ Ok(json!({"id": "before"})),+ Err(failure),+ Ok(json!({"id": "after"})),+ ]));++ let response = frame_stream(stream, WireFormat::OpenAiChat).into_response();+ let body = String::from_utf8(to_bytes(response.into_body(), usize::MAX).await?.to_vec())?;++ // A stream error is terminal: later chunks and success markers must not be emitted.+ assert!(body.contains("before"));+ assert!(body.contains("boom"));+ assert!(!body.contains("after"));+ assert!(!body.contains("[DONE]"));+ Ok(())+ }+}diff --git a/crates/switchyard-server/tests/profile_migration.rs b/crates/switchyard-server/tests/profile_migration.rsdeleted file mode 100644index abcb5cd35..000000000--- a/crates/switchyard-server/tests/profile_migration.rs+++ /dev/null@@ -1,837 +0,0 @@-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-// SPDX-License-Identifier: Apache-2.0--//! Migration tests proving components-v2 profiles cover legacy serving contracts.--use std::io::ErrorKind;-use std::io::{Read, Write};-use std::net::{SocketAddr, TcpListener as StdTcpListener, TcpStream};-use std::sync::{Arc, Mutex};-use std::thread;--use axum::body::Body;-use axum::http::{Request, StatusCode};-use http_body_util::BodyExt;-use serde_json::{json, Value};-use switchyard_components_v2::{- parse_profile_config_str, profile_stats_accumulator, ProfileConfigFormat,-};-use switchyard_server::{build_switchyard_router, ServerState};-use tokio::sync::Mutex as TokioMutex;-use tower::ServiceExt;--static STATS_TEST_LOCK: TokioMutex<()> = TokioMutex::const_new(());--#[tokio::test]-async fn noop_profile_serves_all_inbound_formats_without_upstream() -> TestResult {- let _stats_guard = STATS_TEST_LOCK.lock().await;- reset_stats()?;- let app = build_switchyard_router(state_from_yaml(- r#"-profiles:- bench:- type: noop-"#,- )?);-- let models = app- .clone()- .oneshot(request("GET", "/v1/models", None)?)- .await?;- assert_eq!(models.status(), StatusCode::OK);- assert_eq!(json_body(models).await?["model_pool"], json!(["bench"]));-- let chat = app- .clone()- .oneshot(request(- "POST",- "/v1/chat/completions",- Some(chat_body("bench")),- )?)- .await?;- assert_eq!(chat.status(), StatusCode::OK);- assert_eq!(- json_body(chat).await?["choices"][0]["message"]["content"],- "ok"- );-- let messages = app- .clone()- .oneshot(request(- "POST",- "/v1/messages",- Some(json!({- "model": "bench",- "max_tokens": 16,- "messages": [{"role": "user", "content": "hi"}],- })),- )?)- .await?;- assert_eq!(messages.status(), StatusCode::OK);- assert_eq!(json_body(messages).await?["content"][0]["text"], "ok");-- let responses = app- .oneshot(request(- "POST",- "/v1/responses",- Some(json!({"model": "bench", "input": "hi"})),- )?)- .await?;- assert_eq!(responses.status(), StatusCode::OK);- assert_eq!(- json_body(responses).await?["output"][0]["content"][0]["text"],- "ok"- );- Ok(())-}--#[tokio::test]-async fn passthrough_profile_routes_all_inbound_formats_to_configured_target() -> TestResult {- let _stats_guard = STATS_TEST_LOCK.lock().await;- reset_stats()?;- let Some(stub) = HttpStub::start(vec![- StubResponse::ok(),- StubResponse::ok(),- StubResponse::ok(),- ])?- else {- log_loopback_bind_skip();- return Ok(());- };- let app = build_switchyard_router(state_from_yaml(&format!(- r#"-targets:- direct:- model: provider/direct- format: openai- base_url: {base_url}-profiles:- direct-profile:- type: passthrough- target: direct-"#,- base_url = stub.base_url- ))?);-- let chat = app- .clone()- .oneshot(request(- "POST",- "/v1/chat/completions",- Some(chat_body("direct-profile")),- )?)- .await?;- assert_eq!(chat.status(), StatusCode::OK);- let body = json_body(chat).await?;- assert_eq!(body["choices"][0]["message"]["content"], "stub-ok");-- let messages = app- .clone()- .oneshot(request(- "POST",- "/v1/messages",- Some(json!({- "model": "direct-profile",- "max_tokens": 16,- "messages": [{"role": "user", "content": "hi"}],- })),- )?)- .await?;- assert_eq!(messages.status(), StatusCode::OK);- assert_eq!(json_body(messages).await?["content"][0]["text"], "stub-ok");-- let responses = app- .oneshot(request(- "POST",- "/v1/responses",- Some(json!({"model": "direct-profile", "input": "hi"})),- )?)- .await?;- assert_eq!(responses.status(), StatusCode::OK);- assert_eq!(- json_body(responses).await?["output"][0]["content"][0]["text"],- "stub-ok"- );-- let seen = stub.requests()?;- assert_eq!(seen.len(), 3);- for request in seen {- assert_eq!(request["method"], "POST");- assert_eq!(request["path"], "/v1/chat/completions");- assert_eq!(request["body"]["model"], "provider/direct");- }- Ok(())-}--#[tokio::test]-async fn random_routing_profile_covers_strong_and_weak_selection() -> TestResult {- let _stats_guard = STATS_TEST_LOCK.lock().await;- reset_stats()?;- let cases = [(1.0, "provider/strong"), (0.0, "provider/weak")];- for (strong_probability, expected_model) in cases {- let Some(stub) = HttpStub::start(vec![StubResponse::ok()])? else {- log_loopback_bind_skip();- return Ok(());- };- let app = build_switchyard_router(state_from_yaml(&format!(- r#"-targets:- strong:- model: provider/strong- format: openai- base_url: {base_url}- weak:- model: provider/weak- format: openai- base_url: {base_url}-profiles:- random-profile:- type: random-routing- strong: strong- weak: weak- strong_probability: {strong_probability}- rng_seed: 7-"#,- base_url = stub.base_url- ))?);-- let response = app- .oneshot(request(- "POST",- "/v1/chat/completions",- Some(chat_body("random-profile")),- )?)- .await?;-- assert_eq!(response.status(), StatusCode::OK);- let seen = stub.requests()?;- assert_eq!(seen.len(), 1);- assert_eq!(seen[0]["body"]["model"], expected_model);- }- Ok(())-}--#[tokio::test]-async fn stage_router_profile_threshold_zero_uses_dimensions_signal_path() -> TestResult {- let _stats_guard = STATS_TEST_LOCK.lock().await;- reset_stats()?;- let Some(stub) = HttpStub::start(vec![StubResponse::ok()])? else {- log_loopback_bind_skip();- return Ok(());- };- let app = build_switchyard_router(state_from_yaml(&format!(- r#"-targets:- strong:- model: provider/strong- format: openai- base_url: {base_url}- weak:- model: provider/weak- format: openai- base_url: {base_url}-profiles:- smart-stage_router:- type: stage_router- capable: strong- efficient: weak- fallback_target_on_evict: strong- picker: capable_first- confidence_threshold: 0.0-"#,- base_url = stub.base_url- ))?);-- let response = app- .clone()- .oneshot(request(- "POST",- "/v1/chat/completions",- Some(chat_body("smart-stage_router")),- )?)- .await?;- assert_eq!(response.status(), StatusCode::OK);-- let seen = stub.requests()?;- assert_eq!(seen.len(), 1);- assert_eq!(seen[0]["body"]["model"], "provider/weak");-- let stats = app.oneshot(request("GET", "/v1/stats", None)?).await?;- let stats = json_body(stats).await?;- assert_eq!(stats["routing_decisions"]["stage_router"]["dimensions"], 1);- assert_eq!(stats["classifier"]["total_requests"], 0);- assert_eq!(stats["models"]["provider/weak"]["tier"], "efficient");- Ok(())-}--#[tokio::test]-async fn stage_router_profile_threshold_one_uses_llm_classifier_path() -> TestResult {- let _stats_guard = STATS_TEST_LOCK.lock().await;- reset_stats()?;- let Some(stub) = HttpStub::start(vec![- StubResponse::classifier("capable"),- StubResponse::ok(),- ])?- else {- log_loopback_bind_skip();- return Ok(());- };- let app = build_switchyard_router(state_from_yaml(&format!(- r#"-targets:- strong:- model: provider/strong- format: openai- base_url: {base_url}- weak:- model: provider/weak- format: openai- base_url: {base_url}-profiles:- smart-stage_router:- type: stage_router- capable: strong- efficient: weak- fallback_target_on_evict: strong- picker: capable_first- confidence_threshold: 1.0- classifier:- model: classifier/model- api_key: test-key- base_url: {base_url}- timeout_secs: 5.0-"#,- base_url = stub.base_url- ))?);-- let response = app- .clone()- .oneshot(request(- "POST",- "/v1/chat/completions",- Some(chat_body("smart-stage_router")),- )?)- .await?;- assert_eq!(response.status(), StatusCode::OK);-- let seen = stub.requests()?;- assert_eq!(seen.len(), 2);- assert_eq!(seen[0]["body"]["model"], "classifier/model");- assert_eq!(seen[1]["body"]["model"], "provider/strong");-- let stats = app.oneshot(request("GET", "/v1/stats", None)?).await?;- let stats = json_body(stats).await?;- assert_eq!(- stats["routing_decisions"]["stage_router"]["llm-classifier"],- 1- );- assert_eq!(stats["classifier"]["total_requests"], 1);- assert_eq!(stats["models"]["provider/strong"]["tier"], "capable");- Ok(())-}--#[tokio::test]-async fn stage_router_profile_retries_fallback_after_context_overflow() -> TestResult {- let _stats_guard = STATS_TEST_LOCK.lock().await;- reset_stats()?;- let Some(stub) = HttpStub::start(vec![StubResponse::context_overflow(), StubResponse::ok()])?- else {- log_loopback_bind_skip();- return Ok(());- };- let app = build_switchyard_router(state_from_yaml(&format!(- r#"-targets:- strong:- model: provider/strong- format: openai- base_url: {base_url}- weak:- model: provider/weak- format: openai- base_url: {base_url}-profiles:- smart-stage_router:- type: stage_router- capable: strong- efficient: weak- fallback_target_on_evict: strong- picker: efficient_first- confidence_threshold: 0.7-"#,- base_url = stub.base_url- ))?);-- let response = app- .clone()- .oneshot(request(- "POST",- "/v1/chat/completions",- Some(chat_body("smart-stage_router")),- )?)- .await?;- assert_eq!(response.status(), StatusCode::OK);-- let seen = stub.requests()?;- assert_eq!(seen.len(), 2);- assert_eq!(seen[0]["body"]["model"], "provider/weak");- assert_eq!(seen[1]["body"]["model"], "provider/strong");-- let stats = app.oneshot(request("GET", "/v1/stats", None)?).await?;- let stats = json_body(stats).await?;- assert_eq!(stats["routing_decisions"]["stage_router"]["fall_open"], 1);- assert_eq!(stats["models"]["provider/strong"]["tier"], "capable");- Ok(())-}--#[tokio::test]-async fn latency_service_profile_uses_health_selection_and_retries_failed_target() -> TestResult {- let _stats_guard = STATS_TEST_LOCK.lock().await;- reset_stats()?;- let Some(stub) = HttpStub::start(vec![- StubResponse::latency_health(),- StubResponse::error(503, json!({"error": {"message": "fast unavailable"}})),- StubResponse::ok(),- ])?- else {- log_loopback_bind_skip();- return Ok(());- };- let app = build_switchyard_router(state_from_yaml(&latency_profile_yaml(&stub.origin, 1))?);-- let response = app- .oneshot(request(- "POST",- "/v1/chat/completions",- Some(chat_body("latency-profile")),- )?)- .await?;- assert_eq!(response.status(), StatusCode::OK);- assert_eq!(- json_body(response).await?["choices"][0]["message"]["content"],- "stub-ok"- );-- let seen = stub.requests()?;- assert_eq!(seen.len(), 3);- assert_eq!(seen[0]["method"], "GET");- assert!(seen[0]["path"]- .as_str()- .unwrap_or_default()- .starts_with("/v1/endpoints/health?"));- assert!(seen[0]["path"]- .as_str()- .unwrap_or_default()- .contains("endpoint_ids=fast"));- assert!(seen[0]["path"]- .as_str()- .unwrap_or_default()- .contains("endpoint_ids=slow"));- assert_eq!(seen[1]["path"], "/fast/v1/chat/completions");- assert_eq!(seen[1]["body"]["model"], "provider/fast");- assert_eq!(seen[2]["path"], "/slow/v1/chat/completions");- assert_eq!(seen[2]["body"]["model"], "provider/slow");- Ok(())-}--#[tokio::test]-async fn latency_service_profile_propagates_last_upstream_error_after_retries() -> TestResult {- let _stats_guard = STATS_TEST_LOCK.lock().await;- reset_stats()?;- let Some(stub) = HttpStub::start(vec![- StubResponse::latency_health(),- StubResponse::error(503, json!({"error": {"message": "fast unavailable"}})),- StubResponse::error(502, json!({"error": {"message": "slow unavailable"}})),- ])?- else {- log_loopback_bind_skip();- return Ok(());- };- let app = build_switchyard_router(state_from_yaml(&latency_profile_yaml(&stub.origin, 1))?);-- let response = app- .oneshot(request(- "POST",- "/v1/chat/completions",- Some(chat_body("latency-profile")),- )?)- .await?;- assert_eq!(response.status(), StatusCode::BAD_GATEWAY);- let body = json_body(response).await?;- assert_eq!(body["error"]["type"], "upstream_error");- assert!(body["error"]["message"]- .as_str()- .unwrap_or_default()- .contains("slow unavailable"));-- let seen = stub.requests()?;- assert_eq!(seen.len(), 3);- assert_eq!(seen[1]["body"]["model"], "provider/fast");- assert_eq!(seen[2]["body"]["model"], "provider/slow");- Ok(())-}--#[tokio::test]-async fn profile_config_negative_cases_fail_before_legacy_deletion() -> TestResult {- let unknown_profile_type = state_from_yaml(- r#"-profiles:- bad:- type: not-a-profile-"#,- )- .err()- .ok_or("unknown profile type should fail")?- .to_string();- assert!(unknown_profile_type.contains("unknown profile type"));- assert!(unknown_profile_type.contains("not-a-profile"));-- let unknown_target = state_from_yaml(- r#"-targets: {}-profiles:- bad:- type: passthrough- target: missing-"#,- )- .err()- .ok_or("unknown target should fail")?- .to_string();- assert!(unknown_target.contains("unknown target missing"));-- let invalid_config = state_from_yaml(- r#"-targets:- strong:- model: provider/strong- format: openai- weak:- model: provider/weak- format: openai-profiles:- random-profile:- type: random-routing- strong: strong- weak: weak- strong_probability: 2.0-"#,- )- .err()- .ok_or("invalid random-routing probability should fail")?- .to_string();- assert!(invalid_config.contains("strong_probability"));- Ok(())-}--#[tokio::test]-async fn unknown_inbound_model_stays_a_client_error() -> TestResult {- let app = build_switchyard_router(state_from_yaml(- r#"-profiles:- bench:- type: noop-"#,- )?);-- let response = app- .oneshot(request(- "POST",- "/v1/chat/completions",- Some(chat_body("missing")),- )?)- .await?;-- assert_eq!(response.status(), StatusCode::NOT_FOUND);- assert_eq!(- json_body(response).await?["error"]["code"],- "model_not_found"- );- Ok(())-}--#[derive(Clone)]-struct StubResponse {- status: u16,- body: Value,-}--impl StubResponse {- fn ok() -> Self {- Self {- status: 200,- body: json!({- "id": "chatcmpl-stub",- "object": "chat.completion",- "model": "stub",- "choices": [{- "index": 0,- "message": {"role": "assistant", "content": "stub-ok"},- "finish_reason": "stop",- }],- "usage": {- "prompt_tokens": 1,- "completion_tokens": 1,- "total_tokens": 2,- },- }),- }- }-- fn latency_health() -> Self {- Self {- status: 200,- body: json!({- "endpoint_health": {- "fast": {"status": "healthy", "last_latency_ms": 10.0},- "slow": {"status": "degraded", "last_latency_ms": 1.0},- },- }),- }- }-- fn error(status: u16, body: Value) -> Self {- Self { status, body }- }-- fn classifier(tier: &str) -> Self {- Self {- status: 200,- body: json!({- "id": "chatcmpl-classifier",- "object": "chat.completion",- "model": "classifier/model",- "choices": [{- "index": 0,- "message": {"role": "assistant", "content": json!({"tier": tier}).to_string()},- "finish_reason": "stop",- }],- "usage": {- "prompt_tokens": 3,- "completion_tokens": 1,- "total_tokens": 4,- },- }),- }- }-- fn context_overflow() -> Self {- Self {- status: 400,- body: json!({- "error": {- "code": "context_length_exceeded",- "message": "maximum context length exceeded",- },- }),- }- }-}--/// In-process HTTP stub that serves one response per accepted request.-struct HttpStub {- /// Server origin used by latency-service health polling.- origin: String,- /// OpenAI-compatible base URL ending in `/v1`.- base_url: String,- /// Bound address used to wake blocked accepts during drop.- addr: SocketAddr,- /// Number of requests the stub thread expects before exiting.- expected_requests: usize,- /// Captured method, path, and JSON body for each real request.- requests: Arc<Mutex<Vec<Value>>>,- /// Background accept loop joined on drop after wake-up connects.- handle: Option<thread::JoinHandle<()>>,-}--impl HttpStub {- /// Binds an ephemeral port and returns `None` when loopback is sandbox-denied.- fn start(responses: Vec<StubResponse>) -> TestResult<Option<Self>> {- let listener = match StdTcpListener::bind("127.0.0.1:0") {- Ok(listener) => listener,- Err(error) if error.kind() == ErrorKind::PermissionDenied => return Ok(None),- Err(error) => return Err(error.into()),- };- let addr = listener.local_addr()?;- let expected_requests = responses.len();- let requests = Arc::new(Mutex::new(Vec::new()));- let thread_requests = Arc::clone(&requests);- let handle = thread::spawn(move || {- for response in responses {- let Ok((mut stream, _addr)) = listener.accept() else {- return;- };- if let Ok((method, path, body)) = read_http_request(&mut stream) {- if let Ok(mut requests) = thread_requests.lock() {- requests.push(json!({"method": method, "path": path, "body": body}));- }- }- let body = response.body.to_string();- let reason = if response.status == 200 {- "OK"- } else {- "ERROR"- };- let _ = write!(- stream,- "HTTP/1.1 {} {}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",- response.status,- reason,- body.len(),- body- );- }- });-- Ok(Some(Self {- origin: format!("http://{addr}"),- base_url: format!("http://{addr}/v1"),- addr,- expected_requests,- requests,- handle: Some(handle),- }))- }-- fn requests(&self) -> TestResult<Vec<Value>> {- self.requests- .lock()- .map(|requests| requests.clone())- .map_err(|_| "stub request mutex poisoned".into())- }-}--impl Drop for HttpStub {- fn drop(&mut self) {- // Wake pending accepts before joining the stub thread.- for _ in 0..self.expected_requests {- let _ = TcpStream::connect(self.addr);- }- if let Some(handle) = self.handle.take() {- let _ = handle.join();- }- }-}--/// Reads one small HTTP request, tolerating partial reads until headers and-/// the declared `content-length` body are available.-fn read_http_request(stream: &mut TcpStream) -> TestResult<(String, String, Value)> {- let mut buffer = Vec::new();- let mut header_end = None;- while header_end.is_none() {- let mut chunk = [0; 1024];- let read = stream.read(&mut chunk)?;- if read == 0 {- break;- }- buffer.extend_from_slice(&chunk[..read]);- header_end = find_bytes(&buffer, b"\r\n\r\n").map(|index| index + 4);- }-- let Some(body_start) = header_end else {- return Err("HTTP request headers were incomplete".into());- };- let headers = std::str::from_utf8(&buffer[..body_start])?;- let mut request_line = headers- .lines()- .next()- .ok_or("HTTP request line was missing")?- .split_whitespace();- let method = request_line- .next()- .ok_or("HTTP request line was missing a method")?- .to_string();- let path = request_line- .next()- .ok_or("HTTP request line was missing a path")?- .to_string();- let content_length = headers- .lines()- .find_map(|line| {- let (name, value) = line.split_once(':')?;- name.eq_ignore_ascii_case("content-length")- .then(|| value.trim().parse::<usize>().ok())- .flatten()- })- .unwrap_or(0);-- while buffer.len() < body_start + content_length {- let mut chunk = [0; 1024];- let read = stream.read(&mut chunk)?;- if read == 0 {- break;- }- buffer.extend_from_slice(&chunk[..read]);- }-- if buffer.len() < body_start + content_length {- return Err("HTTP request body was incomplete".into());- }- let body = if content_length == 0 {- Value::Null- } else {- serde_json::from_slice(&buffer[body_start..body_start + content_length])?- };- Ok((method, path, body))-}--fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option<usize> {- haystack- .windows(needle.len())- .position(|window| window == needle)-}--fn state_from_yaml(input: &str) -> TestResult<ServerState> {- let plan = parse_profile_config_str(input, ProfileConfigFormat::Yaml)?.resolve()?;- Ok(ServerState::from_plan(&plan)?)-}--fn chat_body(model: &str) -> Value {- json!({"model": model, "messages": [{"role": "user", "content": "hi"}]})-}--fn latency_profile_yaml(origin: &str, max_retries: usize) -> String {- format!(- r#"-targets:- fast:- model: provider/fast- format: openai- base_url: {origin}/fast/v1- slow:- model: provider/slow- format: openai- base_url: {origin}/slow/v1-profiles:- latency-profile:- type: latency-service- latency_service_url: {origin}- targets: [fast, slow]- max_retries: {max_retries}-"#- )-}--fn request(method: &str, uri: &str, body: Option<Value>) -> TestResult<Request<Body>> {- let builder = Request::builder()- .method(method)- .uri(uri)- .header("content-type", "application/json");- let body = body.map_or_else(Body::empty, |body| Body::from(body.to_string()));- Ok(builder.body(body)?)-}--async fn json_body(response: axum::response::Response) -> TestResult<Value> {- let bytes = response.into_body().collect().await?.to_bytes();- Ok(serde_json::from_slice(&bytes)?)-}--fn reset_stats() -> switchyard_core::Result<()> {- profile_stats_accumulator().reset()-}--fn log_loopback_bind_skip() {- eprintln!("SKIP: permission denied binding loopback socket");-}--type TestResult<T = ()> = std::result::Result<T, Box<dyn std::error::Error + Send + Sync>>;diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex 90ee30b1b..0db025d20 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -1,841 +1,359 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-//! Integration tests for the components-v2 Rust profile server.--use std::io::ErrorKind;-use std::io::{Read, Write};-use std::net::{SocketAddr, TcpListener as StdTcpListener, TcpStream};-use std::sync::{Arc, Mutex, OnceLock};-use std::thread;--use async_trait::async_trait;-use axum::body::Body;-use axum::http::{Request, StatusCode};+//! Integration tests for the libsy Rust server.++use std::collections::{BTreeMap, HashSet};+use std::convert::Infallible;+use std::error::Error;+use std::sync::Arc;++use axum::body::{Body, Bytes};+use axum::extract::State;+use axum::http::{Request as HttpRequest, StatusCode};+use axum::response::sse::{Event, Sse};+use axum::response::{IntoResponse, Response as HttpResponse};+use axum::routing::post;+use axum::{Json, Router};use http_body_util::BodyExt;+use libsy::algorithms::Random;+use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient};use serde_json::{json, Value};-use switchyard_components_v2::{- parse_profile_config_str, profile_stats_accumulator, Profile, ProfileConfigFormat,- ProfileInput, ProfileResponse, RoutingMetadata,-};-use switchyard_core::{- ChatRequestType, ChatResponse, ModelId, Result, StreamEvent, SwitchyardError,-};-use switchyard_server::{build_switchyard_router, ProfileRegistry, ServerState};+use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};+use switchyard_server::{build_switchyard_router, ServerState};+use tokio::net::TcpListener;+use tokio::sync::Mutex;+use tokio::task::JoinHandle;use tower::ServiceExt;-#[tokio::test]-async fn minimal_noop_config_boots_and_serves_core_routes() -> TestResult {- let _stats_guard = stats_guard().await;- reset_stats()?;- let app = build_switchyard_router(state_from_yaml(- r#"-profiles:- bench:- type: noop-"#,- )?);-- let health = app- .clone()- .oneshot(request("GET", "/health", None)?)- .await?;- assert_eq!(health.status(), StatusCode::OK);- assert_eq!(json_body(health).await?, json!({"status": "ok"}));+type TestError = Box<dyn Error + Send + Sync>;+type TestResult<T = ()> = Result<T, TestError>;- let models = app- .clone()- .oneshot(request("GET", "/v1/models", None)?)- .await?;- assert_eq!(models.status(), StatusCode::OK);- let models = json_body(models).await?;- assert_eq!(models["object"], "list");- assert_eq!(models["data"][0]["id"], "bench");- assert_eq!(models["default_model"], "bench");- assert_eq!(models["model_pool"], json!(["bench"]));-- let chat = app- .oneshot(request(- "POST",- "/v1/chat/completions",- Some(json!({- "model": "bench",- "messages": [{"role": "user", "content": "hi"}],- })),- )?)- .await?;- assert_eq!(chat.status(), StatusCode::OK);- assert_eq!(- json_body(chat).await?["choices"][0]["message"]["content"],- "ok"- );- Ok(())-}+const ROUTE_MODEL: &str = "switchyard/random";-#[tokio::test]-async fn all_request_endpoints_route_through_selected_profile() -> TestResult {- let _stats_guard = stats_guard().await;- reset_stats()?;- let app = build_switchyard_router(state_from_yaml(- r#"-profiles:- bench:- type: noop-"#,- )?);-- let anthropic = app- .clone()- .oneshot(request(- "POST",- "/v1/messages",- Some(json!({- "model": "bench",- "max_tokens": 16,- "messages": [{"role": "user", "content": "hi"}],- })),- )?)- .await?;- assert_eq!(anthropic.status(), StatusCode::OK);- let anthropic = json_body(anthropic).await?;- assert_eq!(anthropic["type"], "message");- assert_eq!(anthropic["content"][0]["text"], "ok");-- let responses = app- .oneshot(request(- "POST",- "/v1/responses",- Some(json!({"model": "bench", "input": "hi"})),- )?)- .await?;- assert_eq!(responses.status(), StatusCode::OK);- let responses = json_body(responses).await?;- assert_eq!(responses["object"], "response");- assert_eq!(responses["output"][0]["content"][0]["text"], "ok");- Ok(())+struct MockUpstream {+ base_url: String,+ calls: Arc<Mutex<Vec<Value>>>,+ task: JoinHandle<()>,}-#[tokio::test]-async fn missing_and_unknown_models_return_client_errors() -> TestResult {- let app = build_switchyard_router(state_from_yaml(- r#"-profiles:- bench:- type: noop-"#,- )?);-- let missing = app- .clone()- .oneshot(request(- "POST",- "/v1/chat/completions",- Some(json!({"messages": [{"role": "user", "content": "hi"}]})),- )?)- .await?;- assert_eq!(missing.status(), StatusCode::BAD_REQUEST);- assert_eq!(- json_body(missing).await?["error"]["type"],- "invalid_request_error"- );-- let unknown = app- .oneshot(request(- "POST",- "/v1/chat/completions",- Some(json!({- "model": "missing-route",- "messages": [{"role": "user", "content": "hi"}],- })),- )?)- .await?;- assert_eq!(unknown.status(), StatusCode::NOT_FOUND);- assert_eq!(- json_body(unknown).await?["error"]["type"],- "model_not_found"- );- Ok(())+impl MockUpstream {+ async fn start() -> TestResult<Self> {+ let calls = Arc::new(Mutex::new(Vec::new()));+ let app = Router::new()+ .route("/v1/chat/completions", post(upstream_chat))+ .with_state(Arc::clone(&calls));+ let listener = TcpListener::bind("127.0.0.1:0").await?;+ let addr = listener.local_addr()?;+ let task = tokio::spawn(async move {+ if let Err(error) = axum::serve(listener, app).await {+ tracing::error!(error = %error, "mock upstream stopped");+ }+ });+ Ok(Self {+ base_url: format!("http://{addr}/v1"),+ calls,+ task,+ })+ }}-#[tokio::test]-async fn malformed_and_non_object_json_return_shared_client_errors() -> TestResult {- let app = build_switchyard_router(state_from_yaml(- r#"-profiles:- bench:- type: noop-"#,- )?);-- for uri in ["/v1/chat/completions", "/v1/messages", "/v1/responses"] {- let malformed = app.clone().oneshot(raw_request("POST", uri, "{")?).await?;- assert_eq!(malformed.status(), StatusCode::BAD_REQUEST);- assert_eq!(json_body(malformed).await?["error"]["code"], "invalid_body");-- let non_object = app.clone().oneshot(raw_request("POST", uri, "[]")?).await?;- assert_eq!(non_object.status(), StatusCode::BAD_REQUEST);- assert_eq!(- json_body(non_object).await?["error"]["code"],- "invalid_body"- );+impl Drop for MockUpstream {+ fn drop(&mut self) {+ self.task.abort();}- Ok(())}-#[tokio::test]-async fn translation_errors_do_not_emit_routing_metadata_headers() -> TestResult {- let app = build_switchyard_router(state_from_profile("bad", Arc::new(BadTranslationProfile))?);+async fn upstream_chat(+ State(calls): State<Arc<Mutex<Vec<Value>>>>,+ Json(body): Json<Value>,+) -> HttpResponse {+ calls.lock().await.push(body.clone());+ if body["messages"][0]["content"] == "fail" {+ return (+ StatusCode::IM_A_TEAPOT,+ Json(json!({"error": {"message": "upstream rejected request"}})),+ )+ .into_response();+ }- let response = app- .oneshot(request(- "POST",- "/v1/messages",- Some(json!({- "model": "bad",- "messages": [{"role": "user", "content": "hi"}],- "max_tokens": 8,- })),- )?)- .await?;+ let model = body["model"].as_str().unwrap_or("unknown").to_string();+ if body["stream"].as_bool() == Some(true) {+ let events = [+ json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"role": "assistant"}}]}).to_string(),+ json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "hello"}}]}).to_string(),+ json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}).to_string(),+ "[DONE]".to_string(),+ ];+ let stream = futures_util::stream::iter(+ events+ .into_iter()+ .map(|data| Ok::<Event, Infallible>(Event::default().data(data))),+ );+ return Sse::new(stream).into_response();+ }- assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);- assert!(!response- .headers()- .keys()- .any(|name| name.as_str().starts_with("x-model-router-")));- Ok(())+ Json(json!({+ "id": "chatcmpl-test",+ "object": "chat.completion",+ "model": model,+ "choices": [{+ "index": 0,+ "message": {"role": "assistant", "content": "ok"},+ "finish_reason": "stop"+ }],+ "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}+ }))+ .into_response()+}++fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<ServerState> {+ let backend = Backend::OpenAiChat(HttpBackendConfig {+ base_url: base_url.to_string(),+ api_key: Some("test-key".to_string()),+ extra_headers: BTreeMap::new(),+ });+ let target_models = routes+ .iter()+ .flat_map(|(_, targets)| targets.iter().copied())+ .collect::<HashSet<_>>();+ let model_configs = target_models+ .into_iter()+ .map(|model| ModelConfig::new(model, backend.clone(), None))+ .collect::<Vec<_>>();+ let client: Arc<dyn RoutedLlmClient> = Arc::new(TranslatingLlmClient::new(&model_configs)?);+ let entries = routes.iter().map(|(route_model, targets)| {+ let target_set = LlmTargetSet::new(+ targets+ .iter()+ .map(|model| LlmTarget {+ semantic_name: (*model).to_string(),+ llm_client: Some(Arc::clone(&client)),+ })+ .collect(),+ );+ let algorithm: Arc<dyn Algorithm> = Arc::new(Random::new(target_set));+ ((*route_model).to_string(), algorithm)+ });+ Ok(ServerState::new(entries)?)}-#[tokio::test]-async fn target_with_same_id_and_model_is_registered_once() -> TestResult {- let _stats_guard = stats_guard().await;- let Some(stub) = HttpStub::start(1)? else {- log_loopback_bind_skip();- return Ok(());- };- let app = build_switchyard_router(state_from_yaml(&format!(- r#"-targets:- upstream-direct:- model: upstream-direct- format: openai- base_url: {base_url}-profiles:- direct-profile:- type: passthrough- target: upstream-direct-"#,- base_url = stub.base_url- ))?);-- let models = app- .clone()- .oneshot(request("GET", "/v1/models", None)?)- .await?;- assert_eq!(- json_body(models).await?["model_pool"],- json!(["direct-profile", "upstream-direct"])- );-- let response = app- .oneshot(request(- "POST",- "/v1/chat/completions",- Some(json!({- "model": "upstream-direct",- "messages": [{"role": "user", "content": "hi"}],- })),- )?)- .await?;- assert_eq!(response.status(), StatusCode::OK);- assert_eq!(stub.requests()?[0]["model"], "upstream-direct");- Ok(())+async fn test_app(routes: &[(&str, &[&str])]) -> TestResult<(MockUpstream, Router)> {+ let upstream = MockUpstream::start().await?;+ let app = build_switchyard_router(random_state(&upstream.base_url, routes)?);+ Ok((upstream, app))}-#[tokio::test]-async fn random_routing_profile_reaches_selected_backend_path() -> TestResult {- let _stats_guard = stats_guard().await;- reset_stats()?;- let Some(stub) = HttpStub::start(1)? else {- log_loopback_bind_skip();- return Ok(());+async fn send(app: &Router, method: &str, path: &str, body: Option<Value>) -> TestResult<Response> {+ let mut builder = HttpRequest::builder().method(method).uri(path);+ let request_body = if let Some(body) = body {+ builder = builder.header("content-type", "application/json");+ Body::from(serde_json::to_vec(&body)?)+ } else {+ Body::empty()};- let app = build_switchyard_router(state_from_yaml(&format!(- r#"-targets:- strong:- model: upstream-strong- format: openai- base_url: {base_url}- weak:- model: upstream-weak- format: openai- base_url: {base_url}-profiles:- random:- type: random-routing- strong: strong- weak: weak- strong_probability: 0.0000004- rng_seed: 7-"#,- base_url = stub.base_url- ))?);-- let response = app- .oneshot(request(- "POST",- "/v1/chat/completions",- Some(json!({- "model": "random",- "messages": [{"role": "user", "content": "hi"}],- })),- )?)- .await?;- assert_eq!(response.status(), StatusCode::OK);- for (name, expected) in [- ("x-model-router-selected-model", "upstream-weak"),- ("x-model-router-selected-tier", "weak"),- ("x-model-router-version", "random-routing:v1"),- ("x-model-router-tolerance", "0.0000004"),- ] {- assert_eq!(header(&response, name), Some(expected));- }- assert!(header(&response, "x-model-router-rationale")- .is_some_and(|value| value.contains("strong_probability 0.0000004; selected weak")));+ let response = app.clone().oneshot(builder.body(request_body)?).await?;+ let status = response.status();+ let headers = response.headers().clone();+ let bytes = response.into_body().collect().await?.to_bytes();+ Ok(Response {+ status,+ headers,+ bytes,+ })+}- let seen = stub.requests()?;- assert_eq!(seen.len(), 1);- assert_eq!(seen[0]["model"], "upstream-weak");- Ok(())+struct Response {+ status: StatusCode,+ headers: axum::http::HeaderMap,+ bytes: Bytes,}-#[tokio::test]-async fn latency_service_profile_reaches_configured_backend_path() -> TestResult {- let _stats_guard = stats_guard().await;- reset_stats()?;- let Some(stub) = HttpStub::start(1)? else {- log_loopback_bind_skip();- return Ok(());- };- let app = build_switchyard_router(state_from_yaml(&format!(- r#"-targets:- fast:- model: upstream-fast- format: openai- base_url: {base_url}-profiles:- latency:- type: latency-service- latency_service_url: http://latency.local- targets: [fast]-"#,- base_url = stub.base_url- ))?);-- let response = app- .oneshot(request(- "POST",- "/v1/chat/completions",- Some(json!({- "model": "latency",- "messages": [{"role": "user", "content": "hi"}],- })),- )?)- .await?;- assert_eq!(response.status(), StatusCode::OK);+impl Response {+ fn json(&self) -> TestResult<Value> {+ Ok(serde_json::from_slice(&self.bytes)?)+ }- let seen = stub.requests()?;- assert_eq!(seen.len(), 1);- assert_eq!(seen[0]["model"], "upstream-fast");- Ok(())+ fn text(&self) -> TestResult<&str> {+ Ok(std::str::from_utf8(&self.bytes)?)+ }}#[tokio::test]-async fn stats_endpoints_use_components_v2_global_accumulator() -> TestResult {- let _stats_guard = stats_guard().await;- reset_stats()?;- profile_stats_accumulator().record_success("served-model", Some(12.0), Some("strong"))?;- let app = build_switchyard_router(state_from_yaml(- r#"-profiles:- bench:- type: noop-"#,- )?);-- let stats = app- .clone()- .oneshot(request("GET", "/v1/routing/stats", None)?)- .await?;- assert_eq!(stats.status(), StatusCode::OK);+async fn routes_dispatch_and_discovery_endpoints_are_stable() -> TestResult {+ let (upstream, app) = test_app(&[+ ("switchyard/coding", &["model/code"]),+ ("switchyard/general", &["model/general"]),+ ])+ .await?;++ let health = send(&app, "GET", "/health", None).await?;+ assert_eq!(health.status, StatusCode::OK);+ assert_eq!(health.json()?, json!({"status": "ok"}));++ let models = send(&app, "GET", "/v1/models", None).await?;+ assert_eq!(models.status, StatusCode::OK);assert_eq!(- json_body(stats).await?["models"]["served-model"]["calls"],- 1+ models.json()?["model_pool"],+ json!(["switchyard/coding", "switchyard/general"]));- let reset = app- .clone()- .oneshot(request("POST", "/v1/routing/stats/reset", None)?)- .await?;- assert_eq!(reset.status(), StatusCode::OK);- assert_eq!(json_body(reset).await?, json!({"status": "reset"}));-- let after = app.oneshot(request("GET", "/v1/stats", None)?).await?;- assert_eq!(json_body(after).await?["models"], json!({}));- Ok(())-}+ let missing = send(&app, "GET", "/missing", None).await?;+ assert_eq!(missing.status, StatusCode::NOT_FOUND);+ assert_eq!(missing.json()?["error"]["code"], "endpoint_not_found");-#[tokio::test]-async fn openai_streams_are_sse_framed_with_done() -> TestResult {- let app = build_switchyard_router(state_from_profile(- "stream",- Arc::new(StreamProfile {- kind: StreamKind::OpenAi,- routing_metadata: None,- }),- )?);-- let response = app- .oneshot(request(+ for (route_model, target_model) in [+ ("switchyard/general", "model/general"),+ ("switchyard/coding", "model/code"),+ ] {+ let response = send(+ &app,"POST","/v1/chat/completions",Some(json!({- "model": "stream",- "messages": [{"role": "user", "content": "hi"}],- "stream": true,+ "model": route_model,+ "messages": [{"role": "user", "content": "hi"}]})),- )?)+ ).await?;- assert_eq!(response.status(), StatusCode::OK);- let body = text_body(response).await?;- assert!(body.contains("data: {\"choices\":[{\"delta\":{\"content\":\"hello\"}"));- assert!(body.contains("data: [DONE]"));+ assert_eq!(response.status, StatusCode::OK);+ assert_eq!(+ response+ .headers+ .get("x-model-router-selected-model")+ .and_then(|value| value.to_str().ok()),+ Some(target_model)+ );+ }++ let calls = upstream.calls.lock().await;+ assert_eq!(calls[0]["model"], "model/general");+ assert_eq!(calls[1]["model"], "model/code");Ok(())}#[tokio::test]-async fn openai_streams_include_routing_metadata_headers() -> TestResult {- let app = build_switchyard_router(state_from_profile(- "stream",- Arc::new(StreamProfile {- kind: StreamKind::OpenAi,- routing_metadata: Some(RoutingMetadata {- selected_model: Some("served-model".to_string()),- selected_tier: Some("weak".to_string()),- confidence: Some(0.0000004),- router_version: Some("test-router:v1".to_string()),- tolerance: Some(0.0000004),- rationale: Some("line\nbreak".to_string()),- }),- }),- )?);+async fn all_inbound_formats_run_libsy_and_return_the_caller_format() -> TestResult {+ let (upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;- let response = app- .oneshot(request(- "POST",+ let cases = [+ ("/v1/chat/completions",- Some(json!({- "model": "stream",- "messages": [{"role": "user", "content": "hi"}],- "stream": true,- })),- )?)- .await?;- assert_eq!(response.status(), StatusCode::OK);- for (name, expected) in [- ("x-model-router-selected-model", "served-model"),- ("x-model-router-confidence", "0.0000004"),- ("x-model-router-rationale", "line break"),- ] {- assert_eq!(header(&response, name), Some(expected));- }- let body = text_body(response).await?;- assert!(body.contains("data: [DONE]"));- Ok(())-}--#[tokio::test]-async fn anthropic_streams_are_named_sse_without_done() -> TestResult {- let app = build_switchyard_router(state_from_profile(- "stream",- Arc::new(StreamProfile {- kind: StreamKind::Anthropic,- routing_metadata: None,- }),- )?);-- let response = app- .oneshot(request(- "POST",+ json!({+ "model": ROUTE_MODEL,+ "messages": [{"role": "user", "content": "hi"}]+ }),+ ),+ ("/v1/messages",- Some(json!({- "model": "stream",+ json!({+ "model": ROUTE_MODEL,"max_tokens": 16,- "messages": [{"role": "user", "content": "hi"}],- "stream": true,- })),- )?)- .await?;- assert_eq!(response.status(), StatusCode::OK);- let body = text_body(response).await?;- assert!(body.contains("event: message_start"));- assert!(body.contains("\"type\":\"message_start\""));- assert!(!body.contains("[DONE]"));- Ok(())-}+ "messages": [{"role": "user", "content": "hi"}]+ }),+ ),+ (+ "/v1/responses",+ json!({"model": ROUTE_MODEL, "input": "hi"}),+ ),+ ];-#[tokio::test]-async fn endpoint_metadata_is_passed_to_profiles() -> TestResult {- let captured = Arc::new(Mutex::new(None));- let app = build_switchyard_router(state_from_profile(- "capture",- Arc::new(CaptureProfile {- captured: Arc::clone(&captured),- }),- )?);-- let response = app- .oneshot(- Request::builder()- .method("POST")- .uri("/v1/messages")- .header("content-type", "application/json")- .header("X-Request-ID", "req-123")- .header("X-Switchyard-Trace", "trace-a")- .body(Body::from(- json!({- "model": "capture",- "max_tokens": 16,- "messages": [{"role": "user", "content": "hi"}],- })- .to_string(),- ))?,- )- .await?;- assert_eq!(response.status(), StatusCode::OK);-- let input = captured- .lock()- .map_err(|_| "captured input mutex poisoned")?- .clone()- .ok_or("profile should have received input")?;- assert_eq!(input.request.model(), Some("capture"));- assert_eq!(- input.metadata.request_id.as_ref().map(|id| id.as_str()),- Some("req-123")- );+ let mut responses = Vec::new();+ for (path, body) in cases {+ responses.push(send(&app, "POST", path, Some(body)).await?);+ }++ assert!(responses+ .iter()+ .all(|response| response.status == StatusCode::OK));assert_eq!(- input.metadata.inbound_format,- Some(ChatRequestType::Anthropic)+ responses[0].json()?["choices"][0]["message"]["content"],+ "ok");+ assert_eq!(responses[1].json()?["content"][0]["text"], "ok");assert_eq!(- input- .metadata- .headers- .get("x-switchyard-trace")- .map(Vec::as_slice),- Some(&["trace-a".to_string()][..])+ responses[2].json()?["output"][0]["content"][0]["text"],+ "ok");- Ok(())-}--/// Test profile that emits one deterministic stream event for SSE framing checks.-#[derive(Clone)]-struct StreamProfile {- kind: StreamKind,- routing_metadata: Option<RoutingMetadata>,-}--/// Stream format variant emitted by `StreamProfile`.-#[derive(Clone, Copy)]-enum StreamKind {- OpenAi,- Anthropic,-}--#[async_trait]-impl Profile for StreamProfile {- async fn run(&self, _input: ProfileInput) -> Result<ProfileResponse> {- let response = match self.kind {- StreamKind::OpenAi => ChatResponse::OpenAiStream(Box::pin(futures_util::stream::iter(- [Ok(StreamEvent::Json(json!({- "id": "chatcmpl-test",- "object": "chat.completion.chunk",- "model": "served-model",- "choices": [{- "index": 0,- "delta": {"content": "hello"},- "finish_reason": null,- }],- })))],- ))),- StreamKind::Anthropic => ChatResponse::AnthropicStream(Box::pin(- futures_util::stream::iter([Ok(StreamEvent::Json(json!({- "type": "message_start",- "message": {- "id": "msg-test",- "type": "message",- "role": "assistant",- "content": [],- "model": "claude-test",- "stop_reason": null,- "stop_sequence": null,- "usage": {"input_tokens": 1, "output_tokens": 0},- },- })))]),- )),- };- Ok(match &self.routing_metadata {- Some(metadata) => ProfileResponse::with_routing_metadata(response, metadata.clone()),- None => ProfileResponse::from(response),- })- }-}--struct CaptureProfile {- captured: Arc<Mutex<Option<ProfileInput>>>,-}--struct BadTranslationProfile;--#[async_trait]-impl Profile for BadTranslationProfile {- async fn run(&self, _input: ProfileInput) -> Result<ProfileResponse> {- Ok(ProfileResponse::with_routing_metadata(- ChatResponse::openai_completion(json!("not an OpenAI response object")),- RoutingMetadata {- selected_model: Some("bad-upstream".to_string()),- selected_tier: Some("weak".to_string()),- router_version: Some("test-router:v1".to_string()),- ..RoutingMetadata::default()- },- ))- }-}--#[async_trait]-impl Profile for CaptureProfile {- async fn run(&self, input: ProfileInput) -> Result<ProfileResponse> {- {- let mut captured = self- .captured- .lock()- .map_err(|_| SwitchyardError::Other("captured input mutex poisoned".to_string()))?;- *captured = Some(input);- }- Ok(ChatResponse::anthropic_completion(json!({- "id": "msg-capture",- "type": "message",- "role": "assistant",- "content": [{"type": "text", "text": "captured"}],- "model": "capture",- "stop_reason": "end_turn",- "stop_sequence": null,- "usage": {"input_tokens": 1, "output_tokens": 1},- }))- .into())- }-}--/// In-process HTTP stub that records JSON request bodies from backend calls.-struct HttpStub {- base_url: String,- addr: SocketAddr,- expected_requests: usize,- requests: Arc<Mutex<Vec<Value>>>,- handle: Option<thread::JoinHandle<()>>,-}--impl HttpStub {- /// Binds an ephemeral port and accepts the expected number of stub requests.- fn start(expected_requests: usize) -> TestResult<Option<Self>> {- let listener = match StdTcpListener::bind("127.0.0.1:0") {- Ok(listener) => listener,- Err(error) if error.kind() == ErrorKind::PermissionDenied => return Ok(None),- Err(error) => return Err(error.into()),- };- let addr = listener.local_addr()?;- let requests = Arc::new(Mutex::new(Vec::new()));- let thread_requests = Arc::clone(&requests);- let handle = thread::spawn(move || {- for _ in 0..expected_requests {- let Ok((mut stream, _addr)) = listener.accept() else {- return;- };- if let Ok(body) = read_http_body(&mut stream) {- if let Ok(value) = serde_json::from_slice::<Value>(&body) {- if let Ok(mut requests) = thread_requests.lock() {- requests.push(value);- }- }- }- let response = json!({- "id": "chatcmpl-stub",- "object": "chat.completion",- "model": "stub",- "choices": [{- "index": 0,- "message": {"role": "assistant", "content": "stub-ok"},- "finish_reason": "stop",- }],- "usage": {- "prompt_tokens": 1,- "completion_tokens": 1,- "total_tokens": 2,- },- })- .to_string();- let _ = write!(- stream,- "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",- response.len(),- response- );- }- });-- Ok(Some(Self {- base_url: format!("http://{addr}/v1"),- addr,- expected_requests,- requests,- handle: Some(handle),- }))- }-- /// Returns the JSON request bodies captured by the stub thread.- fn requests(&self) -> TestResult<Vec<Value>> {- self.requests- .lock()- .map(|requests| requests.clone())- .map_err(|_| "stub request mutex poisoned".into())- }-}--impl Drop for HttpStub {- fn drop(&mut self) {- // Wake pending accepts before joining the stub thread.- for _ in 0..self.expected_requests {- let _ = TcpStream::connect(self.addr);- }- if let Some(handle) = self.handle.take() {- let _ = handle.join();- }- }-}--fn read_http_body(stream: &mut std::net::TcpStream) -> TestResult<Vec<u8>> {- let mut buffer = Vec::new();- let mut header_end = None;- while header_end.is_none() {- let mut chunk = [0; 1024];- let read = stream.read(&mut chunk)?;- if read == 0 {- break;- }- buffer.extend_from_slice(&chunk[..read]);- header_end = find_bytes(&buffer, b"\r\n\r\n").map(|index| index + 4);- }-- let Some(body_start) = header_end else {- return Err("HTTP request headers were incomplete".into());- };- let headers = std::str::from_utf8(&buffer[..body_start])?;- let content_length = headers- .lines()- .find_map(|line| {- let (name, value) = line.split_once(':')?;- name.eq_ignore_ascii_case("content-length")- .then(|| value.trim().parse::<usize>().ok())- .flatten()- })- .unwrap_or(0);-- while buffer.len() < body_start + content_length {- let mut chunk = [0; 1024];- let read = stream.read(&mut chunk)?;- if read == 0 {- break;- }- buffer.extend_from_slice(&chunk[..read]);+ for response in &responses {+ assert_eq!(+ response+ .headers+ .get("x-model-router-selected-model")+ .and_then(|value| value.to_str().ok()),+ Some("model/a")+ );}- Ok(buffer[body_start..body_start + content_length].to_vec())-}--fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option<usize> {- haystack- .windows(needle.len())- .position(|window| window == needle)-}--fn state_from_yaml(input: &str) -> TestResult<ServerState> {- let plan = parse_profile_config_str(input, ProfileConfigFormat::Yaml)?.resolve()?;- Ok(ServerState::from_plan(&plan)?)-}--fn state_from_profile(model: &'static str, profile: Arc<dyn Profile>) -> TestResult<ServerState> {- let registry = ProfileRegistry::from_profiles([(- ModelId::from_static(model),- profile,- model.to_string(),- )])?;- Ok(ServerState::new(registry))-}--fn request(method: &str, uri: &str, body: Option<Value>) -> TestResult<Request<Body>> {- let builder = Request::builder()- .method(method)- .uri(uri)- .header("content-type", "application/json");- let body = body.map_or_else(Body::empty, |body| Body::from(body.to_string()));- Ok(builder.body(body)?)-}--fn raw_request(method: &str, uri: &str, body: &str) -> TestResult<Request<Body>> {- let builder = Request::builder()- .method(method)- .uri(uri)- .header("content-type", "application/json");- Ok(builder.body(Body::from(body.to_string()))?)-}--async fn json_body(response: axum::response::Response) -> TestResult<Value> {- let bytes = response.into_body().collect().await?.to_bytes();- Ok(serde_json::from_slice(&bytes)?)-}--async fn text_body(response: axum::response::Response) -> TestResult<String> {- let bytes = response.into_body().collect().await?.to_bytes();- Ok(String::from_utf8(bytes.to_vec())?)-}--fn header<'a>(response: &'a axum::response::Response, name: &str) -> Option<&'a str> {- response- .headers()- .get(name)- .and_then(|value| value.to_str().ok())+ let calls = upstream.calls.lock().await;+ assert_eq!(calls.len(), 3);+ assert!(calls.iter().all(|call| call["model"] == "model/a"));+ Ok(())}-fn reset_stats() -> Result<()> {- profile_stats_accumulator().reset()+#[tokio::test]+async fn streaming_response_is_framed_for_the_inbound_api() -> TestResult {+ let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;++ let response = send(+ &app,+ "POST",+ "/v1/chat/completions",+ Some(json!({+ "model": ROUTE_MODEL,+ "messages": [{"role": "user", "content": "hi"}],+ "stream": true+ })),+ )+ .await?;++ assert_eq!(response.status, StatusCode::OK);+ assert!(response.text()?.contains("hello"));+ assert!(response.text()?.contains("data: [DONE]"));+ Ok(())}-/// Serializes tests that touch the global profile stats accumulator.-///-/// The lock is initialized once for the test process and held by each caller-/// until the returned guard is dropped.-async fn stats_guard() -> tokio::sync::MutexGuard<'static, ()> {- static LOCK: OnceLock<tokio::sync::Mutex<()>> = OnceLock::new();- LOCK.get_or_init(|| tokio::sync::Mutex::new(()))- .lock()- .await-}+#[tokio::test]+async fn request_and_upstream_errors_use_the_canonical_envelope() -> TestResult {+ let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;++ let unknown = send(+ &app,+ "POST",+ "/v1/chat/completions",+ Some(json!({+ "model": "other",+ "messages": [{"role": "user", "content": "hi"}]+ })),+ )+ .await?;+ assert_eq!(unknown.status, StatusCode::NOT_FOUND);+ assert_eq!(unknown.json()?["error"]["code"], "model_not_found");++ let missing_model = send(+ &app,+ "POST",+ "/v1/chat/completions",+ Some(json!({"messages": [{"role": "user", "content": "hi"}]})),+ )+ .await?;+ assert_eq!(missing_model.status, StatusCode::BAD_REQUEST);+ assert_eq!(+ missing_model.json()?["error"]["code"],+ "invalid_request_error"+ );-fn log_loopback_bind_skip() {- eprintln!("SKIP: permission denied binding loopback socket");+ let upstream_error = send(+ &app,+ "POST",+ "/v1/chat/completions",+ Some(json!({+ "model": ROUTE_MODEL,+ "messages": [{"role": "user", "content": "fail"}]+ })),+ )+ .await?;+ assert_eq!(upstream_error.status, StatusCode::IM_A_TEAPOT);+ assert_eq!(upstream_error.json()?["error"]["code"], "upstream_error");+ Ok(())}--type TestResult<T = ()> = std::result::Result<T, Box<dyn std::error::Error + Send + Sync>>;diff --git a/docs/cli_reference.md b/docs/cli_reference.mdindex c9b47cda1..4c3b98f4a 100644--- a/docs/cli_reference.md+++ b/docs/cli_reference.md@@ -21,7 +21,7 @@ These apply to the top-level `switchyard` command, before any verb.|---|---|| `--version` | Print the installed Switchyard version (`switchyard X.Y.Z`) and exit. Reads the version from the installed package metadata. || `--routing-profiles PATH` / `-c PATH` | Deprecated legacy [Routing](#routing) bundle applied to `serve`, `launch`, and `configure`. Pass before the verb; separate with `--` for clarity. |-| `--enable-rl-logging` | Write local [RL trace logs](#rl-trace-logging) (one `message_history` JSON file per turn) for `launch` and `serve` route-bundle sessions. Pass before the verb: `switchyard --enable-rl-logging launch claude`. Rejected by `serve --config` (the Rust profile server has no Python processor chain). |+| `--enable-rl-logging` | Write local [RL trace logs](#rl-trace-logging) (one `message_history` JSON file per turn) for `launch` and `serve` route-bundle sessions. Pass before the verb: `switchyard --enable-rl-logging launch claude`. Rejected by `serve --config`. || `--rl-log-dir DIR` | Output directory for `--enable-rl-logging` traces (default: `./rl_data`). No effect without `--enable-rl-logging`. |## Cross-cutting flag families@@ -147,8 +147,7 @@ matches the pre-1.0 trace format:```Turns without an assistant choice (e.g. upstream errors) are skipped. The flag-works on `launch` and on `serve` with a route bundle; `serve --config` (the Rust-profile server) rejects it, since it has no Python processor chain to attach to.+works on `launch` and on `serve` with a route bundle; `serve --config` rejects it.### Transport (server verbs)@@ -205,7 +204,7 @@ because the built-in default is `https://openrouter.ai/api/v1`.## `switchyard serve`-Serve a long-running proxy from a Switchyard v2 **profile config**: one YAML/JSON/TOML file declaring `endpoints`, `targets`, and `profiles`. Files whose profiles are all Rust-defined use the Rust profile server. Files that include Python-defined profiles use the Python FastAPI adapter, while keeping the same endpoint paths. Each profile id and each target id is exposed as a model on `GET /v1/models`, so a client selects a profile by setting the request `model` to that id.+Serve a long-running proxy from a Switchyard v2 **profile config**: one YAML/JSON/TOML file declaring `endpoints`, `targets`, and `profiles`. Rust-defined and Python-defined profiles run through the same Python HTTP application and expose the same endpoint paths. Each profile id and each target id is exposed as a model on `GET /v1/models`, so a client selects a profile by setting the request `model` to that id.The server exposes the OpenAI Chat Completions (`/v1/chat/completions`), Anthropic Messages (`/v1/messages`), and OpenAI Responses (`/v1/responses`) APIs on the same host and port.diff --git a/docs/internal/metrics_reference.md b/docs/internal/metrics_reference.mdindex 859c3a9fd..3dcdc18f4 100644--- a/docs/internal/metrics_reference.md+++ b/docs/internal/metrics_reference.md@@ -16,7 +16,7 @@ a drop-in scrape config and starter alert rules.!!! note "Availability: legacy route-bundle serve only"`GET /metrics` is served by the **legacy route-bundle path**- (`switchyard --routing-profiles PATH serve`). The recommended v2 profile server+ (`switchyard --routing-profiles PATH serve`). The v2 profile-config path(`switchyard serve --config`) returns **404** for `/metrics` — use `GET /v1/stats`(or its alias `GET /v1/routing/stats`) on that path instead.diff --git a/switchyard/cli/switchyard_cli.py b/switchyard/cli/switchyard_cli.pyindex cac0ff3d3..a7fb6ebc0 100644--- a/switchyard/cli/switchyard_cli.py+++ b/switchyard/cli/switchyard_cli.py@@ -99,7 +99,6 @@ from switchyard.server.server_util import (add_transport_args,build_and_serve,load_secrets,- resolve_port,resolve_rl_log_dir,)@@ -144,17 +143,6 @@ def _warn_deprecated_saved_route_bundle() -> None:)-def _warn_deprecated_python_profile_server(python_profiles: Sequence[str]) -> None:- _print_deprecation_warning(- "Python-defined profile serving",- details=(- "This config is using the Python FastAPI adapter.",- f"Python profile(s): {', '.join(python_profiles)}.",- "Prefer Rust-defined components-v2 profiles for new serve configs.",- ),- )--class _IntakeEnabledAction(argparse.Action):"""Store the normalized Intake enable flag and warn on the deprecated alias."""@@ -468,7 +456,7 @@ def _cmd_serve(args: argparse.Namespace) -> None:def _cmd_serve_profile_config(args: argparse.Namespace) -> None:- """Serve a components-v2 profile config."""+ """Serve a v2 profile config through the Python HTTP application."""if args.routing_profiles:raise SystemExit("serve --config cannot be combined with --routing-profiles; "@@ -494,47 +482,14 @@ def _cmd_serve_profile_config(args: argparse.Namespace) -> None:raise SystemExit("serve --config does not support Intake options yet.")if getattr(args, "enable_rl_logging", False):raise SystemExit(- "serve --config does not support --enable-rl-logging: the Rust "- "profile server has no Python processor chain to attach the trace "- "logger to. Use serve --routing-profiles for local RL trace logging."+ "serve --config does not support --enable-rl-logging. "+ "Use serve --routing-profiles for local RL trace logging.")- # Inspect first so files containing only Rust-defined profiles can use the- # Rust server path, while files with Python-defined profiles use FastAPI.- from switchyard.lib.profiles.loader import python_profile_ids-- try:- python_profiles = python_profile_ids(args.config)- except Exception as exc:- raise SystemExit(- "serve --config: failed to inspect profile config for "- f"Python-defined profiles: {exc}"- ) from exc- if python_profiles:- _cmd_serve_mixed_profile_config(args, python_profiles)- return-- from switchyard_rust.server import run_profile_server-- port = args.port if isinstance(args.port, int) else resolve_port()- logger.info(- "Switchyard components-v2 profile config loaded from %s",- args.config,- )- run_profile_server(args.config, args.host, port)---def _cmd_serve_mixed_profile_config(- args: argparse.Namespace,- python_profiles: list[str],-) -> None:- """Serve a config containing Python-defined profiles through FastAPI."""- _warn_deprecated_python_profile_server(python_profiles)table = _profile_config_route_table(args.config)logger.info(- "Switchyard profile config loaded from %s with Python-defined profile(s): %s",+ "Switchyard profile config loaded from %s",args.config,- ", ".join(python_profiles),)build_and_serve(args, table, inbound_default="both")@@ -928,10 +883,7 @@ def _build_parser() -> argparse.ArgumentParser:default=None,metavar="PATH",help=(- "Path to a Switchyard v2 profile config (YAML, JSON, or TOML). "- "Files containing only Rust-defined profiles use the Rust profile "- "server; files with Python-defined profiles use the Python FastAPI "- "adapter."+ "Path to a Switchyard v2 profile config (YAML, JSON, or TOML)."),)serve.add_argument(diff --git a/switchyard/lib/processors/random_routing_request_processor.py b/switchyard/lib/processors/random_routing_request_processor.pyindex 48e2e8abf..e9320ef84 100644--- a/switchyard/lib/processors/random_routing_request_processor.py+++ b/switchyard/lib/processors/random_routing_request_processor.py@@ -15,8 +15,6 @@ class RandomRoutingRequestProcessor:"""Rewrite requests to the randomly selected strong or weak target.This is a compatibility component for the current Python profile chain.- New Rust serving code uses components-v2 profiles directly instead of- exporting a Rust request-processor object."""def __init__(self, config: RandomRoutingProcessorConfig) -> None:diff --git a/switchyard/lib/profiles/loader.py b/switchyard/lib/profiles/loader.pyindex 713615be1..ee56925be 100644--- a/switchyard/lib/profiles/loader.py+++ b/switchyard/lib/profiles/loader.py@@ -29,7 +29,7 @@ from switchyard_rust.profiles import (parse_profile_config_path,)-__all__ = ["load_profiles", "load_profiles_and_targets", "python_profile_ids"]+__all__ = ["load_profiles", "load_profiles_and_targets"]_PROFILE_TARGET = "profile_target"@@ -66,17 +66,6 @@ def load_profiles_and_targets(return built, _targets(plan)-def python_profile_ids(path: str | Path) -> list[str]:- """Return the ids of Python-defined profiles declared in ``path``.-- Used by ``serve --config`` to select the Rust server for files containing- only Rust-defined profiles and the Python FastAPI adapter for files that- include Python-defined profiles.- """- _register_shipped_python_profiles()- return sorted(_python_profiles(parse_profile_config_path(path)))--class _RustProfileRunner:"""Adapts an erased Rust ``Profile`` to the ``run(input)`` runner contract."""diff --git a/switchyard_rust/core.py b/switchyard_rust/core.pyindex 3633cfc2c..a7859405e 100644--- a/switchyard_rust/core.py+++ b/switchyard_rust/core.py@@ -252,15 +252,6 @@ class _NativeModule(Protocol):SessionCache: type[Any]session_key_from_body: Any- def run_profile_server(- self,- config_path: str,- host: str,- port: int,- backlog: int,- dry_run: bool,- ) -> None: ...-def _ensure_switchyard_version_env() -> None:if os.environ.get("SWITCHYARD_VERSION", "").strip():diff --git a/switchyard_rust/server.py b/switchyard_rust/server.pydeleted file mode 100644index 7ad6c82c1..000000000--- a/switchyard_rust/server.py+++ /dev/null@@ -1,20 +0,0 @@-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-# SPDX-License-Identifier: Apache-2.0--"""Rust components-v2 profile server bindings."""--from switchyard_rust.core import _load_native---def run_profile_server(- config_path: str,- host: str = "127.0.0.1",- port: int = 4000,- backlog: int = 65_535,- dry_run: bool = False,-) -> None:- """Run the Rust components-v2 profile server from an installed package."""- _load_native().run_profile_server(config_path, host, port, backlog, dry_run)---__all__ = ["run_profile_server"]diff --git a/tests/test_profile_migration.py b/tests/test_profile_migration.pyindex e0a14d94f..58446567c 100644--- a/tests/test_profile_migration.py+++ b/tests/test_profile_migration.py@@ -3,13 +3,11 @@"""Migration tests for replacing legacy serving paths with components-v2 profiles."""-import argparseimport errnoimport jsonimport threadingfrom collections.abc import Iteratorfrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServer-from pathlib import Pathfrom typing import Anyimport pytest@@ -535,73 +533,3 @@ async def test_profile_runner_uses_same_profile_config(assert response.body["model"] == "provider/direct"assert response.body["mock_path"] == "/v2-direct/v1/chat/completions"assert [call["body"]["model"] for call in openai_stub.requests] == ["provider/direct"]---def test_cli_serve_config_delegates_to_rust_profile_server(- mocker: MockerFixture,- tmp_path: Path,-) -> None:- import switchyard.cli.switchyard_cli as cli- import switchyard_rust.server as rust_server-- config_path = tmp_path / "profiles.yaml"- config_path.write_text("profiles:\n bench:\n type: noop\n", encoding="utf-8")- captured: dict[str, object] = {}-- def _fake_run_profile_server(- config_path: str,- host: str = "127.0.0.1",- port: int = 4000,- backlog: int = 65_535,- dry_run: bool = False,- ) -> None:- captured["config_path"] = config_path- captured["host"] = host- captured["port"] = port- captured["backlog"] = backlog- captured["dry_run"] = dry_run-- mocker.patch.object(- rust_server,- "run_profile_server",- side_effect=_fake_run_profile_server,- )-- def _fail_build_and_serve(- args: argparse.Namespace,- switchyard: object,- inbound_default: str = "openai",- disable_backend_streaming: bool = False,- extra_endpoints: list[object] | None = None,- ) -> None:- _ = (- args,- switchyard,- inbound_default,- disable_backend_streaming,- extra_endpoints,- )- pytest.fail("route-bundle server should not run")-- mocker.patch.object(cli, "build_and_serve", side_effect=_fail_build_and_serve)-- args = cli._build_parser().parse_args(- [- "serve",- "--config",- str(config_path),- "--host",- "127.0.0.1",- "--port",- "4555",- ]- )- args.func(args)-- assert captured == {- "config_path": str(config_path),- "host": "127.0.0.1",- "port": 4555,- "backlog": 65_535,- "dry_run": False,- }diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.pyindex 86860c729..cafe39e10 100644--- a/tests/test_route_bundle.py+++ b/tests/test_route_bundle.py@@ -421,73 +421,6 @@ def test_serve_subcommand_hands_table_to_server(assert captured["inbound_default"] == "both"-def test_serve_config_delegates_to_rust_profile_server(- mocker: Any,- tmp_path,-) -> None:- import switchyard.cli.switchyard_cli as cli- import switchyard_rust.server as rust_server-- config_path = tmp_path / "profiles.yaml"- config_path.write_text("profiles:\n bench:\n type: noop\n")- captured: dict[str, Any] = {}-- def _fake_run_profile_server(- config_path: str,- host: str = "127.0.0.1",- port: int = 4000,- backlog: int = 65_535,- dry_run: bool = False,- ) -> None:- captured["config_path"] = config_path- captured["host"] = host- captured["port"] = port- captured["backlog"] = backlog- captured["dry_run"] = dry_run-- mocker.patch.object(- rust_server,- "run_profile_server",- side_effect=_fake_run_profile_server,- )-- def _fail_build_and_serve(- args: argparse.Namespace,- switchyard: object,- *,- inbound_default: str = "openai",- disable_backend_streaming: bool = False,- **_kwargs: object,- ) -> None:- _ = (args, switchyard, inbound_default, disable_backend_streaming)- pytest.fail("route-bundle server should not run")-- mocker.patch.object(- cli,- "build_and_serve",- side_effect=_fail_build_and_serve,- )-- args = cli._build_parser().parse_args([- "serve",- "--config",- str(config_path),- "--host",- "127.0.0.1",- "--port",- "4555",- ])- args.func(args)-- assert captured == {- "config_path": str(config_path),- "host": "127.0.0.1",- "port": 4555,- "backlog": 65_535,- "dry_run": False,- }--def test_serve_config_and_routing_profiles_are_mutually_exclusive(tmp_path,) -> None:diff --git a/tests/test_serve_profile_config.py b/tests/test_serve_profile_config.pyindex 29809fe34..299cb4efa 100644--- a/tests/test_serve_profile_config.py+++ b/tests/test_serve_profile_config.py@@ -1,13 +1,7 @@# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.# SPDX-License-Identifier: Apache-2.0-"""Tests for serving a v2 profile config via `serve --config`.--`dry_run` exercises the full load -> resolve -> build -> registry path of the-Rust profile server without binding a socket. Files with Python-defined profiles-use the Python FastAPI adapter so both profile implementations are routable on-the same paths.-"""+"""Tests for serving a v2 profile config via `serve --config`."""import argparseimport json@@ -20,36 +14,7 @@ import pytestfrom fastapi.testclient import TestClientfrom switchyard.cli.switchyard_cli import _cmd_serve_profile_config, _profile_config_route_table-from switchyard.lib.profiles.loader import python_profile_idsfrom switchyard.server.switchyard_app import build_switchyard_app-from switchyard_rust.core import SwitchyardConfigError-from switchyard_rust.server import run_profile_server--_RUST_CONFIG = """-targets:- strong:- model: provider/strong- format: openai- base_url: http://127.0.0.1:9/strong/v1- api_key: test-key- weak:- model: provider/weak- format: openai- base_url: http://127.0.0.1:9/weak/v1- api_key: test-key--profiles:- fast:- type: passthrough- target: weak- smart-stage-router:- type: stage_router- capable: strong- efficient: weak- fallback_target_on_evict: strong- picker: capable_first- confidence_threshold: 0.7-"""_PYTHON_CONFIG = """targets:@@ -155,36 +120,9 @@ def _write(tmp_path: Path, text: str, name: str = "profiles.yaml") -> Path:return path-def test_dry_run_validates_rust_profiles_and_direct_targets(tmp_path: Path) -> None:- # passthrough + stage_router + the two targets (directly addressable).- path = _write(tmp_path, _RUST_CONFIG)- # dry_run loads, resolves, and builds the registry, then returns without- # binding a socket. No exception == a servable config.- run_profile_server(str(path), dry_run=True)---def test_dry_run_rejects_invalid_config(tmp_path: Path) -> None:- path = _write(tmp_path, "profiles:\n bad:\n type: passthrough\n target: ghost\n")- with pytest.raises(SwitchyardConfigError):- run_profile_server(str(path), dry_run=True)---def test_shipped_example_config_is_servable(monkeypatch: pytest.MonkeyPatch) -> None:- # Guards examples/profiles.yaml from rotting; dry_run never connects upstream.- monkeypatch.setenv("OPENROUTER_API_KEY", "dummy-key")- example = Path(__file__).resolve().parents[1] / "examples" / "profiles.yaml"- run_profile_server(str(example), dry_run=True)---def test_python_profile_ids_classifies_config(tmp_path: Path) -> None:- assert python_profile_ids(_write(tmp_path, _PYTHON_CONFIG)) == ["smart"]- assert python_profile_ids(_write(tmp_path, _RUST_CONFIG, name="rust.yaml")) == []---def test_serve_config_uses_fastapi_for_python_profiles(+def test_serve_config_uses_fastapi_profile_table(monkeypatch: pytest.MonkeyPatch,tmp_path: Path,- capsys: pytest.CaptureFixture[str],) -> None:path = _write(tmp_path, _PYTHON_CONFIG)args = _serve_args(path)@@ -211,10 +149,6 @@ def test_serve_config_uses_fastapi_for_python_profiles(assert captured["models"] == ["smart", "weak", "provider/weak"]assert captured["inbound_default"] == "both"- stderr = capsys.readouterr().err- assert "warning: Python-defined profile serving is deprecated." in stderr- assert "Python FastAPI adapter" in stderr- assert "Python profile(s): smart." in stderrdef test_profile_config_route_table_serves_mixed_profiles(@@ -285,24 +219,6 @@ def test_profile_config_route_table_serves_mixed_profiles()-def test_serve_config_fails_closed_when_python_profile_inspection_fails(- monkeypatch: pytest.MonkeyPatch,- tmp_path: Path,-) -> None:- path = _write(tmp_path, _RUST_CONFIG)-- def fail_inspection(_path: str) -> list[str]:- raise RuntimeError("inspection exploded")-- monkeypatch.setattr(- "switchyard.lib.profiles.loader.python_profile_ids",- fail_inspection,- )-- with pytest.raises(SystemExit, match="failed to inspect.*inspection exploded"):- _cmd_serve_profile_config(_serve_args(path))--def _serve_args(path: Path) -> argparse.Namespace:return argparse.Namespace(config=str(path),