Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

refactor(server): serve libsy algorithms without profiles (#98)

nachiketb-nvidiae09b5b032d5f
diff --git a/Cargo.lock b/Cargo.lock
index 0370a7e5e..f45b55c22 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1747,7 +1747,6 @@ dependencies = [
"switchyard-components-v2",
"switchyard-core",
"switchyard-libsy",
- "switchyard-server",
"switchyard-translation",
"tokio",
"tracing",
@@ -1765,10 +1764,9 @@ dependencies = [
"futures-util",
"http-body-util",
"rustls",
- "serde",
"serde_json",
- "switchyard-components-v2",
- "switchyard-core",
+ "switchyard-libsy",
+ "switchyard-llm-client",
"switchyard-translation",
"tokio",
"tower",
diff --git a/crates/switchyard-py/Cargo.toml b/crates/switchyard-py/Cargo.toml
index 074625a67..560ea2f65 100644
--- a/crates/switchyard-py/Cargo.toml
+++ b/crates/switchyard-py/Cargo.toml
@@ -27,7 +27,6 @@ serde_json = "1"
switchyard-core = { path = "../switchyard-core" }
switchyard-components = { path = "../switchyard-components" }
switchyard-components-v2 = { path = "../switchyard-components-v2" }
-switchyard-server = { path = "../switchyard-server" }
switchyard-translation = { path = "../switchyard-translation" }
tokio = { version = "1", features = ["rt", "rt-multi-thread", "sync"] }
tracing = { version = "0.1", default-features = false, features = ["std"] }
diff --git a/crates/switchyard-py/src/lib.rs b/crates/switchyard-py/src/lib.rs
index 6b3fe73d3..5ef43c27a 100644
--- a/crates/switchyard-py/src/lib.rs
+++ b/crates/switchyard-py/src/lib.rs
@@ -9,7 +9,6 @@ mod errors;
mod libsy_bindings;
mod profile_bindings;
mod py_serde;
-mod server_bindings;
mod translation;
#[pymodule]
@@ -20,6 +19,5 @@ fn _switchyard_rust(module: &Bound<'_, PyModule>) -> PyResult<()> {
component_bindings::register(module)?;
libsy_bindings::register(module)?;
profile_bindings::register(module)?;
- server_bindings::register(module)?;
Ok(())
}
diff --git a/crates/switchyard-py/src/server_bindings.rs b/crates/switchyard-py/src/server_bindings.rs
deleted file mode 100644
index a1b5cec87..000000000
--- a/crates/switchyard-py/src/server_bindings.rs
+++ /dev/null
@@ -1,60 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Python bindings for the Rust components-v2 profile server.
-
-use std::net::{IpAddr, SocketAddr};
-use std::path::PathBuf;
-
-use pyo3::exceptions::PyValueError;
-use pyo3::prelude::*;
-use switchyard_server::{run_server, ServerRunOptions, DEFAULT_LISTEN_BACKLOG};
-
-use crate::errors::py_core_error;
-
-/// Run the Rust components-v2 profile server from Python.
-#[pyfunction]
-#[pyo3(signature = (
- config_path,
- host = "127.0.0.1",
- port = 4000,
- backlog = DEFAULT_LISTEN_BACKLOG,
- dry_run = false,
-))]
-fn run_profile_server(
- py: Python<'_>,
- config_path: String,
- host: &str,
- port: u16,
- backlog: u32,
- dry_run: bool,
-) -> PyResult<()> {
- let ip: IpAddr = host.parse().map_err(|error| {
- PyValueError::new_err(format!(
- "host must be an IP address accepted by the Rust server, got {host:?}: {error}"
- ))
- })?;
- let options = ServerRunOptions {
- config: PathBuf::from(config_path),
- addr: SocketAddr::new(ip, port),
- backlog,
- dry_run,
- tls: None,
- };
-
- // `detach` runs synchronously with the GIL released, so startup errors still
- // return to the Python caller instead of disappearing into a background task.
- py.detach(move || {
- let runtime = tokio::runtime::Builder::new_multi_thread()
- .enable_all()
- .build()
- .map_err(|error| PyValueError::new_err(error.to_string()))?;
- runtime.block_on(run_server(options)).map_err(py_core_error)
- })
-}
-
-/// Registers Rust server bindings with the native Python module.
-pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
- module.add_function(wrap_pyfunction!(run_profile_server, module)?)?;
- Ok(())
-}
diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.toml
index 23012b853..88e88ac02 100644
--- a/crates/switchyard-server/Cargo.toml
+++ b/crates/switchyard-server/Cargo.toml
@@ -4,7 +4,7 @@
[package]
name = "switchyard-server"
version = "0.1.0"
-description = "Rust HTTP server surface for Switchyard profiles"
+description = "Rust HTTP server surface for libsy algorithms"
authors.workspace = true
edition.workspace = true
license.workspace = true
@@ -18,10 +18,9 @@ axum-server = { version = "0.8", features = ["tls-rustls"] }
rustls = { version = "0.23" }
clap = { version = "4", features = ["derive", "env"] }
futures-util = "0.3"
-serde = { version = "1", features = ["derive"] }
+libsy = { package = "switchyard-libsy", path = "../libsy" }
+switchyard-llm-client = { path = "../libsy-llm-client" }
serde_json = "1"
-switchyard-components-v2 = { path = "../switchyard-components-v2" }
-switchyard-core = { path = "../switchyard-core" }
switchyard-translation = { path = "../switchyard-translation" }
tokio = { version = "1", features = ["macros", "net", "rt-multi-thread", "signal"] }
tracing = { version = "0.1", default-features = false, features = ["std"] }
diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md
new file mode 100644
index 000000000..4bed0fd07
--- /dev/null
+++ b/crates/switchyard-server/README.md
@@ -0,0 +1,15 @@
+# switchyard-server
+
+`switchyard-server` exposes libsy algorithms through OpenAI Chat Completions, OpenAI Responses,
+and Anthropic Messages endpoints. The current CLI builds uniform-random routes over one shared
+upstream backend.
+
+```bash
+cargo run -p switchyard-server -- \
+ --route switchyard/general=model/a,model/b \
+ --route switchyard/coding=model/c,model/d \
+ --base-url https://example.com/v1 \
+ --api-key "$API_KEY"
+```
+
+Select a route by sending its `switchyard/*` ID as the request's `model`.
diff --git a/crates/switchyard-server/src/cli.rs b/crates/switchyard-server/src/cli.rs
index d71008ce6..63e88345e 100644
--- a/crates/switchyard-server/src/cli.rs
+++ b/crates/switchyard-server/src/cli.rs
@@ -1,53 +1,132 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! CLI entrypoint for running the components-v2 Rust profile server.
+//! CLI entrypoint for running the libsy server with random routing.
+use std::collections::{BTreeMap, BTreeSet};
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::path::PathBuf;
+use std::str::FromStr;
+use std::sync::Arc;
-use clap::Parser;
-use switchyard_core::{Result, SwitchyardError};
-use switchyard_server::{run_server, ServerRunOptions, TLSOptions, DEFAULT_LISTEN_BACKLOG};
+use clap::{Parser, ValueEnum};
+use libsy::algorithms::Random;
+use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient};
+use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
+use switchyard_server::{
+ run_server, ServerError, ServerResult, ServerRunOptions, ServerState, TlsOptions,
+ DEFAULT_LISTEN_BACKLOG,
+};
const DEFAULT_HOST: IpAddr = IpAddr::V4(Ipv4Addr::UNSPECIFIED);
const DEFAULT_PORT: u16 = 4000;
+#[derive(Clone, Debug)]
+struct RandomRouteSpec {
+ model: String,
+ targets: Vec<String>,
+}
+
+impl FromStr for RandomRouteSpec {
+ type Err = String;
+
+ fn from_str(value: &str) -> Result<Self, Self::Err> {
+ let (model, target_list) = value
+ .split_once('=')
+ .ok_or_else(|| "route must use MODEL=TARGET[,TARGET...]".to_string())?;
+ let model = model.trim();
+ if model.is_empty() {
+ return Err("route model must not be empty".to_string());
+ }
+ let targets = target_list
+ .split(',')
+ .map(str::trim)
+ .map(str::to_string)
+ .collect::<Vec<_>>();
+ if targets.iter().any(String::is_empty) {
+ return Err(format!("route {model} must contain non-empty targets"));
+ }
+ if targets.iter().collect::<BTreeSet<_>>().len() != targets.len() {
+ return Err(format!("route {model} must contain unique targets"));
+ }
+ Ok(Self {
+ model: model.to_string(),
+ targets,
+ })
+ }
+}
+
+#[derive(Clone, Copy, Debug, ValueEnum)]
+enum UpstreamFormat {
+ #[value(name = "openai-chat")]
+ OpenAiChat,
+ #[value(name = "openai-responses")]
+ OpenAiResponses,
+ #[value(name = "anthropic")]
+ Anthropic,
+}
+
+impl UpstreamFormat {
+ fn backend(self, config: HttpBackendConfig) -> Backend {
+ match self {
+ Self::OpenAiChat => Backend::OpenAiChat(config),
+ Self::OpenAiResponses => Backend::OpenAiResponses(config),
+ Self::Anthropic => Backend::Anthropic(config),
+ }
+ }
+}
+
/// Command-line arguments accepted by the Rust server binary.
#[derive(Debug, Parser)]
#[command(
name = "switchyard-server",
- about = "Run the Rust Switchyard server from a components-v2 profile config",
+ about = "Run uniform random routing with libsy",
version
)]
pub(crate) struct ServerArgs {
- /// Path to a components-v2 profile config file.
- #[arg(short, long, env = "SWITCHYARD_PROFILE_CONFIG", value_name = "PATH")]
- pub(crate) config: PathBuf,
+ /// Random route as MODEL=TARGET[,TARGET...]. Repeat to serve multiple routes.
+ #[arg(
+ long = "route",
+ required = true,
+ value_name = "MODEL=TARGET[,TARGET...]"
+ )]
+ routes: Vec<RandomRouteSpec>,
+
+ /// Base URL shared by the configured upstream targets.
+ #[arg(long, env = "SWITCHYARD_UPSTREAM_BASE_URL")]
+ base_url: String,
+
+ /// Upstream API key. Omit when the backend needs no authentication.
+ #[arg(long, env = "SWITCHYARD_UPSTREAM_API_KEY")]
+ api_key: Option<String>,
+
+ /// Provider wire format used by the upstream backend.
+ #[arg(long, value_enum, default_value = "openai-chat")]
+ upstream_format: UpstreamFormat,
/// Host address to bind.
#[arg(long, default_value_t = DEFAULT_HOST)]
- pub(crate) host: IpAddr,
+ host: IpAddr,
/// Port to bind.
#[arg(short, long, default_value_t = DEFAULT_PORT)]
- pub(crate) port: u16,
+ port: u16,
/// TCP listen backlog passed to the socket before Axum accepts traffic.
#[arg(long, default_value_t = DEFAULT_LISTEN_BACKLOG)]
- pub(crate) backlog: u32,
+ backlog: u32,
- /// Validate and build the config without starting the HTTP listener.
+ /// Validate the algorithm and client configuration without binding a socket.
#[arg(long)]
- pub(crate) dry_run: bool,
+ dry_run: bool,
- /// TLS certificate path, PEM format
+ /// TLS certificate path in PEM format.
#[arg(long, requires = "tls_key")]
- pub(crate) tls_cert: Option<PathBuf>,
+ tls_cert: Option<PathBuf>,
- /// TLS certificate key path, PEM format
+ /// TLS private-key path in PEM format.
#[arg(long, requires = "tls_cert")]
- pub(crate) tls_key: Option<PathBuf>,
+ tls_key: Option<PathBuf>,
}
impl ServerArgs {
@@ -56,30 +135,108 @@ impl ServerArgs {
Self::parse()
}
- fn into_options(self) -> Result<ServerRunOptions> {
- let mut tls_options = None;
- if let (Some(cert), Some(key)) = (self.tls_cert, self.tls_key) {
- if !cert.exists() || !key.exists() {
- return Err(SwitchyardError::InvalidConfig(format!(
- "Invalid path in --tls-cert {} or --tls-key {}. File does not exist.",
- cert.display(),
- key.display()
- )));
+ fn into_runtime(self) -> ServerResult<(ServerState, ServerRunOptions)> {
+ if self.base_url.trim().is_empty() {
+ return Err(ServerError::new("--base-url must not be empty"));
+ }
+
+ let backend = self.upstream_format.backend(HttpBackendConfig {
+ base_url: self.base_url,
+ api_key: self.api_key,
+ extra_headers: BTreeMap::new(),
+ });
+ let target_models = self
+ .routes
+ .iter()
+ .flat_map(|route| route.targets.iter())
+ .collect::<BTreeSet<_>>();
+ let model_configs = target_models
+ .into_iter()
+ .map(|model| ModelConfig::new(model.as_str(), backend.clone(), None))
+ .collect::<Vec<_>>();
+ let client: Arc<dyn RoutedLlmClient> = Arc::new(
+ TranslatingLlmClient::new(&model_configs)
+ .map_err(|error| ServerError::new(error.to_string()))?,
+ );
+ let routes = self.routes.into_iter().map(|route| {
+ let target_set = LlmTargetSet::new(
+ route
+ .targets
+ .iter()
+ .map(|model| LlmTarget {
+ semantic_name: model.clone(),
+ llm_client: Some(Arc::clone(&client)),
+ })
+ .collect(),
+ );
+ let algorithm: Arc<dyn Algorithm> = Arc::new(Random::new(target_set));
+ (route.model, algorithm)
+ });
+ let state = ServerState::new(routes)?;
+
+ let tls = match (self.tls_cert, self.tls_key) {
+ (Some(cert), Some(key)) => {
+ if !cert.exists() || !key.exists() {
+ return Err(ServerError::new(format!(
+ "invalid --tls-cert {} or --tls-key {}: file does not exist",
+ cert.display(),
+ key.display()
+ )));
+ }
+ Some(TlsOptions { cert, key })
}
- tls_options = Some(TLSOptions { cert, key })
+ _ => None,
};
- Ok(ServerRunOptions {
- config: self.config,
+ let options = ServerRunOptions {
addr: SocketAddr::new(self.host, self.port),
backlog: self.backlog,
dry_run: self.dry_run,
- tls: tls_options,
- })
+ tls,
+ };
+ Ok((state, options))
}
}
-/// Loads config, optionally validates it, then starts the Rust server.
-pub(crate) async fn run(args: ServerArgs) -> Result<()> {
- let opts = args.into_options()?;
- run_server(opts).await
+/// Builds the random algorithm and starts the server.
+pub(crate) async fn run(args: ServerArgs) -> ServerResult<()> {
+ let (state, options) = args.into_runtime()?;
+ run_server(state, options).await
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn runtime(routes: &[&str]) -> ServerResult<ServerState> {
+ let mut args = vec!["switchyard-server", "--base-url", "http://127.0.0.1:9/v1"];
+ for route in routes {
+ args.extend(["--route", route]);
+ }
+ let args = ServerArgs::try_parse_from(args)
+ .map_err(|error| ServerError::new(error.to_string()))?;
+ args.into_runtime().map(|(state, _)| state)
+ }
+
+ #[test]
+ fn routes_build_independent_algorithms() -> ServerResult<()> {
+ let state = runtime(&[
+ "switchyard/general=model/a,model/b",
+ "switchyard/coding=model/c,model/d",
+ ])?;
+ assert_eq!(
+ state.models().collect::<Vec<_>>(),
+ ["switchyard/coding", "switchyard/general"]
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn invalid_routes_are_rejected() {
+ for routes in [
+ &["switchyard/general=model/a,model/a"][..],
+ &["switchyard/general=model/a", "switchyard/general=model/b"],
+ ] {
+ assert!(runtime(routes).is_err());
+ }
+ }
}
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index 50595aaab..a49e4e354 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -1,19 +1,16 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! Rust HTTP server surface for components-v2 profile configs.
-//!
-//! The serving path is profile-native: config files load into
-//! `ProfileConfigPlan`, plans build `Profile` runtimes, and HTTP requests
-//! call `Profile::run()` directly.
+//! Rust HTTP server for libsy algorithms.
-mod registry;
mod response;
mod sse;
use std::collections::BTreeMap;
+use std::error::Error;
+use std::fmt::{Display, Formatter};
use std::net::SocketAddr;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
@@ -23,88 +20,108 @@ use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
use axum::{Json, Router};
use axum_server::tls_rustls::RustlsConfig;
+use libsy::{Algorithm, Context, Decision, Metadata, Request};
use serde_json::{json, Value};
+use switchyard_llm_client::LlmClientError;
use tokio::net::{TcpListener, TcpSocket};
-use switchyard_components_v2::{
- parse_profile_config_path, profile_stats_accumulator, ProfileConfigPlan, ProfileInput,
- ProfileResponse, RequestMetadata, RoutingMetadata,
-};
-use switchyard_core::{ChatRequest, ChatRequestType, RequestId, Result, SwitchyardError};
-use switchyard_translation::{TranslationEngine, TranslationPolicy, WireFormat};
+use switchyard_translation::{decode_request, WireFormat};
-pub use registry::{ProfileRegistry, ServedModel};
-
-use crate::response::{translate_chain_response, TranslatedResponse};
+use crate::response::into_http_response;
/// Default TCP listen backlog used by the Rust server.
pub const DEFAULT_LISTEN_BACKLOG: u32 = 65_535;
const HEADER_SELECTED_MODEL: &str = "x-model-router-selected-model";
-const HEADER_SELECTED_TIER: &str = "x-model-router-selected-tier";
-const HEADER_CONFIDENCE: &str = "x-model-router-confidence";
-const HEADER_ROUTER_VERSION: &str = "x-model-router-version";
-const HEADER_TOLERANCE: &str = "x-model-router-tolerance";
const HEADER_RATIONALE: &str = "x-model-router-rationale";
const MAX_ROUTING_HEADER_VALUE_LEN: usize = 512;
+type BoxError = Box<dyn Error + Send + Sync>;
+
+/// Error returned while configuring or running the server.
+#[derive(Debug)]
+pub struct ServerError {
+ message: String,
+}
+
+impl ServerError {
+ /// Creates a server error with a user-facing message.
+ pub fn new(message: impl Into<String>) -> Self {
+ Self {
+ message: message.into(),
+ }
+ }
+}
+
+impl Display for ServerError {
+ fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
+ formatter.write_str(&self.message)
+ }
+}
+
+impl Error for ServerError {}
+
+/// Result returned by server setup and lifecycle operations.
+pub type ServerResult<T> = std::result::Result<T, ServerError>;
+
/// Shared server state used by all endpoint handlers.
#[derive(Clone)]
pub struct ServerState {
- registry: Arc<ProfileRegistry>,
- translation: Arc<TranslationEngine>,
- translation_policy: TranslationPolicy,
+ routes: Arc<BTreeMap<String, Arc<dyn Algorithm>>>,
}
impl ServerState {
- /// Creates server state for a profile registry.
- pub fn new(registry: ProfileRegistry) -> Self {
- Self {
- registry: Arc::new(registry),
- translation: Arc::new(TranslationEngine::default()),
- translation_policy: TranslationPolicy::default(),
+ /// Creates server state from route model IDs and their libsy algorithms.
+ pub fn new(
+ routes: impl IntoIterator<Item = (String, Arc<dyn Algorithm>)>,
+ ) -> ServerResult<Self> {
+ let mut entries = BTreeMap::new();
+ for (model, algorithm) in routes {
+ let model = model.trim();
+ if model.is_empty() {
+ return Err(ServerError::new("route model must not be empty"));
+ }
+ if entries.insert(model.to_string(), algorithm).is_some() {
+ return Err(ServerError::new(format!("duplicate route model {model}")));
+ }
}
+ if entries.is_empty() {
+ return Err(ServerError::new("at least one algorithm route is required"));
+ }
+ Ok(Self {
+ routes: Arc::new(entries),
+ })
}
- /// Builds server state from a resolved profile config plan.
- pub fn from_plan(plan: &ProfileConfigPlan) -> Result<Self> {
- Ok(Self::new(ProfileRegistry::from_plan(plan)?))
- }
-
- /// Returns the profile registry used by this server.
- pub fn registry(&self) -> &ProfileRegistry {
- self.registry.as_ref()
+ /// Returns the route model IDs served by the configured algorithms.
+ pub fn models(&self) -> impl Iterator<Item = &str> {
+ self.routes.keys().map(String::as_str)
}
- /// Dispatches one request to the profile selected by its `model` field.
- async fn run_profile(&self, input: ProfileInput) -> Result<ProfileResponse> {
- let profile = self.registry.lookup(input.request.model())?;
- profile.run(input).await
+ fn algorithm_for_model(&self, model: &str) -> Option<Arc<dyn Algorithm>> {
+ self.routes.get(model).map(Arc::clone)
}
}
-/// Runtime options shared by the Rust binary and Python binding.
+/// Runtime options shared by server entry points.
#[derive(Clone, Debug)]
pub struct ServerRunOptions {
- /// Path to the components-v2 profile config file.
- pub config: PathBuf,
/// Socket address to bind.
pub addr: SocketAddr,
/// TCP listen backlog.
pub backlog: u32,
- /// Validate and print public model IDs without binding a socket.
+ /// Validate runtime construction without binding a socket.
pub dry_run: bool,
- /// If this is set server is HTTPS.
- /// Caller must validate that the files exist.
- pub tls: Option<TLSOptions>,
+ /// TLS certificate configuration, when HTTPS is enabled.
+ pub tls: Option<TlsOptions>,
}
+/// TLS certificate paths used by the server.
#[derive(Clone, Debug)]
-pub struct TLSOptions {
- /// TLS certificate path, PEM format
+pub struct TlsOptions {
+ /// TLS certificate path in PEM format.
pub cert: PathBuf,
-
- /// TLS certificate key path, PEM format
+ /// TLS private-key path in PEM format.
pub key: PathBuf,
}
@@ -114,19 +131,10 @@ impl ServerRunOptions {
}
}
-/// Builds a server state by loading and resolving a profile config path.
-fn state_from_config_path(path: impl AsRef<Path>) -> Result<ServerState> {
- let document = parse_profile_config_path(path)?;
- let plan = document.resolve()?;
- ServerState::from_plan(&plan)
-}
-
-/// Entry point to this module.
-/// Loads config, optionally validates it, then starts the Rust server.
-pub async fn run_server(options: ServerRunOptions) -> Result<()> {
- let state = state_from_config_path(&options.config)?;
+/// Validates the runtime and starts the HTTP server unless `dry_run` is set.
+pub async fn run_server(state: ServerState, options: ServerRunOptions) -> ServerResult<()> {
if options.dry_run {
- println!("{}", dry_run_summary(&options.config, state.registry()));
+ println!("{}", dry_run_summary(&state));
return Ok(());
}
@@ -136,7 +144,7 @@ pub async fn run_server(options: ServerRunOptions) -> Result<()> {
addr: bound_addr,
..options
};
- eprintln!("{}", startup_banner(&server_options, state.registry()));
+ eprintln!("{}", startup_banner(&server_options, &state));
let router = build_switchyard_router(state);
if let Some(tls) = server_options.tls {
serve_tls(listener, router, tls).await
@@ -145,11 +153,9 @@ pub async fn run_server(options: ServerRunOptions) -> Result<()> {
}
}
-async fn serve_tls(listener: TcpListener, router: Router, tls: TLSOptions) -> Result<()> {
- // aws_lc_rs is the default but other crates pull in `ring` also,
- // so rustls doesn't know which one to use. Tell it.
- if let Err(e) = rustls::crypto::aws_lc_rs::default_provider().install_default() {
- eprintln!("TLS crypto provider already installed: {e:?}");
+async fn serve_tls(listener: TcpListener, router: Router, tls: TlsOptions) -> ServerResult<()> {
+ if let Err(error) = rustls::crypto::aws_lc_rs::default_provider().install_default() {
+ tracing::debug!(?error, "TLS crypto provider was already installed");
}
let config = RustlsConfig::from_pem_file(tls.cert, tls.key)
@@ -160,46 +166,38 @@ async fn serve_tls(listener: TcpListener, router: Router, tls: TLSOptions) -> Re
let shutdown_handle = handle.clone();
tokio::spawn(async move {
shutdown_signal().await;
- // Drain connections. Later should loop on connection_count() > 0.
shutdown_handle.graceful_shutdown(Some(Duration::from_secs(2)));
});
let std_listener = listener.into_std().map_err(server_io_error)?;
axum_server::from_tcp_rustls(std_listener, config)
.map_err(server_io_error)?
- .handle(handle.clone())
+ .handle(handle)
.serve(router.into_make_service())
.await
.map_err(server_io_error)
}
-/// Serves a Switchyard router on an already-bound TCP listener.
-async fn serve(listener: TcpListener, router: Router) -> Result<()> {
+async fn serve(listener: TcpListener, router: Router) -> ServerResult<()> {
axum::serve(listener, router)
.with_graceful_shutdown(shutdown_signal())
.await
.map_err(server_io_error)
}
-/// Builds an Axum router with the same primary endpoint paths as the Python app.
-/// Public so that integration tests can see it.
+/// Builds an Axum router for the supported LLM wire formats.
pub fn build_switchyard_router(state: ServerState) -> Router {
Router::new()
.route("/v1/chat/completions", post(openai_chat_completions))
.route("/v1/messages", post(anthropic_messages))
.route("/v1/responses", post(openai_responses))
.route("/v1/models", get(models))
- // Keep the legacy routing stats aliases wired to the same handlers.
- .route("/v1/stats", get(stats))
- .route("/v1/stats/reset", post(reset_stats))
- .route("/v1/routing/stats", get(stats))
- .route("/v1/routing/stats/reset", post(reset_stats))
.route("/health", get(health))
.fallback(not_found)
.with_state(state)
}
-fn bind_tcp_listener(addr: SocketAddr, backlog: u32) -> Result<TcpListener> {
+fn bind_tcp_listener(addr: SocketAddr, backlog: u32) -> ServerResult<TcpListener> {
let socket = if addr.is_ipv4() {
TcpSocket::new_v4()
} else {
@@ -212,8 +210,8 @@ fn bind_tcp_listener(addr: SocketAddr, backlog: u32) -> Result<TcpListener> {
socket.listen(backlog).map_err(server_io_error)
}
-fn server_io_error(error: std::io::Error) -> SwitchyardError {
- SwitchyardError::Other(error.to_string())
+fn server_io_error(error: std::io::Error) -> ServerError {
+ ServerError::new(error.to_string())
}
async fn shutdown_signal() {
@@ -231,17 +229,7 @@ async fn openai_chat_completions(
headers: HeaderMap,
body: std::result::Result<Json<Value>, JsonRejection>,
) -> Response {
- let body = match llm_json_body(body) {
- Ok(body) => body,
- Err(response) => return *response,
- };
- handle_llm_request(
- state,
- ChatRequest::openai_chat(body),
- WireFormat::OpenAiChat,
- metadata_from_headers(&headers, ChatRequestType::OpenAiChat),
- )
- .await
+ handle_endpoint(state, headers, body, WireFormat::OpenAiChat).await
}
async fn anthropic_messages(
@@ -249,116 +237,127 @@ async fn anthropic_messages(
headers: HeaderMap,
body: std::result::Result<Json<Value>, JsonRejection>,
) -> Response {
- let body = match llm_json_body(body) {
- Ok(body) => body,
- Err(response) => return *response,
- };
- handle_llm_request(
- state,
- ChatRequest::anthropic(body),
- WireFormat::AnthropicMessages,
- metadata_from_headers(&headers, ChatRequestType::Anthropic),
- )
- .await
+ handle_endpoint(state, headers, body, WireFormat::AnthropicMessages).await
}
async fn openai_responses(
State(state): State<ServerState>,
headers: HeaderMap,
body: std::result::Result<Json<Value>, JsonRejection>,
+) -> Response {
+ handle_endpoint(state, headers, body, WireFormat::OpenAiResponses).await
+}
+
+async fn handle_endpoint(
+ state: ServerState,
+ headers: HeaderMap,
+ body: std::result::Result<Json<Value>, JsonRejection>,
+ wire_format: WireFormat,
) -> Response {
let body = match llm_json_body(body) {
Ok(body) => body,
- Err(response) => return *response,
+ Err(message) => return invalid_body_error(message),
};
- handle_llm_request(
- state,
- ChatRequest::openai_responses(body),
- WireFormat::OpenAiResponses,
- metadata_from_headers(&headers, ChatRequestType::OpenAiResponses),
- )
- .await
+ handle_llm_request(state, headers, body, wire_format).await
}
fn llm_json_body(
body: std::result::Result<Json<Value>, JsonRejection>,
-) -> std::result::Result<Value, Box<Response>> {
+) -> std::result::Result<Value, String> {
match body {
Ok(Json(value)) if value.is_object() => Ok(value),
- Ok(_) => Err(Box::new(invalid_body_error(
- "Request body must be a JSON object",
- ))),
- Err(error) => Err(Box::new(invalid_body_error(format!(
- "Request body must be valid JSON: {error}"
- )))),
+ Ok(_) => Err("Request body must be a JSON object".to_string()),
+ Err(error) => Err(format!("Request body must be valid JSON: {error}")),
}
}
async fn handle_llm_request(
state: ServerState,
- request: ChatRequest,
- target_format: WireFormat,
- metadata: RequestMetadata,
+ headers: HeaderMap,
+ body: Value,
+ wire_format: WireFormat,
) -> Response {
- if let Err(error) = request.validate() {
- return llm_error(error);
- }
+ let llm_request = match decode_request(wire_format, &body) {
+ Ok(request) => request,
+ Err(error) => return invalid_body_error(error.to_string()),
+ };
+ let Some(requested_model) = llm_request
+ .model
+ .clone()
+ .filter(|model| !model.trim().is_empty())
+ else {
+ return error_response(
+ StatusCode::BAD_REQUEST,
+ "request body must include a non-empty string `model`",
+ "invalid_request_error",
+ "invalid_request_error",
+ );
+ };
+ let Some(algorithm) = state.algorithm_for_model(&requested_model) else {
+ return error_response(
+ StatusCode::NOT_FOUND,
+ format!("No route registered for model {requested_model}"),
+ "model_not_found",
+ "model_not_found",
+ );
+ };
- let profile_response = match state.run_profile(ProfileInput { request, metadata }).await {
- Ok(response) => response,
- Err(error) => return llm_error(error),
+ let request = Request {
+ llm_request,
+ raw_request: Some(body),
+ metadata: Some(metadata_from_headers(&headers)),
};
- let (response, routing_metadata) = profile_response.into_parts();
-
- let mut response = match translate_chain_response(
- response,
- target_format,
- Arc::clone(&state.translation),
- state.translation_policy.clone(),
- ) {
- Ok(TranslatedResponse::Buffered(body)) => Json(body).into_response(),
- Ok(TranslatedResponse::Stream(stream)) => stream.into_response(),
- Err(error) => return server_error(error.to_string()),
+ let (trace, response) = match algorithm.run(Context::default(), request).await {
+ Ok(result) => result,
+ Err(error) => return algorithm_error(error),
};
- attach_routing_metadata_headers(&mut response, routing_metadata.as_ref());
- response
-}
-fn attach_routing_metadata_headers(response: &mut Response, metadata: Option<&RoutingMetadata>) {
- let Some(metadata) = metadata else {
- return;
+ let mut response = match into_http_response(response, wire_format, Some(requested_model)) {
+ Ok(response) => response,
+ Err(error) => return server_error(error.to_string()),
};
- for (name, value) in routing_metadata_headers(metadata) {
- let name = HeaderName::from_static(name);
- let Ok(value) = HeaderValue::from_str(&value) else {
- continue;
- };
- response.headers_mut().insert(name, value);
+ if let Some(decision) = trace.last() {
+ attach_routing_headers(&mut response, decision.as_ref());
}
+ response
}
-fn routing_metadata_headers(metadata: &RoutingMetadata) -> Vec<(&'static str, String)> {
- [
- (HEADER_SELECTED_MODEL, text_header(&metadata.selected_model)),
- (HEADER_SELECTED_TIER, text_header(&metadata.selected_tier)),
- (HEADER_CONFIDENCE, number_header(metadata.confidence)),
- (HEADER_ROUTER_VERSION, text_header(&metadata.router_version)),
- (HEADER_TOLERANCE, number_header(metadata.tolerance)),
- (HEADER_RATIONALE, text_header(&metadata.rationale)),
- ]
- .into_iter()
- .filter_map(|(name, value)| value.map(|value| (name, value)))
- .collect()
+fn metadata_from_headers(headers: &HeaderMap) -> Metadata {
+ let headers = normalized_headers(headers);
+ let mut metadata = Metadata::from_headers(&headers);
+ metadata.http_headers = Some(headers);
+ metadata
}
-fn text_header(value: &Option<String>) -> Option<String> {
- value.as_deref().and_then(sanitize_routing_header_value)
+fn normalized_headers(headers: &HeaderMap) -> BTreeMap<String, String> {
+ headers
+ .iter()
+ .filter_map(|(name, value)| {
+ value
+ .to_str()
+ .ok()
+ .map(|value| (name.as_str().to_ascii_lowercase(), value.to_string()))
+ })
+ .collect()
+}
+
+fn attach_routing_headers(response: &mut Response, decision: &dyn Decision) {
+ insert_routing_header(response, HEADER_SELECTED_MODEL, decision.selected_model());
+ if let Some(reasoning) = decision.reasoning() {
+ insert_routing_header(response, HEADER_RATIONALE, reasoning);
+ }
}
-fn number_header(value: Option<f64>) -> Option<String> {
- value
- .filter(|value| value.is_finite())
- .map(|value| value.to_string())
+fn insert_routing_header(response: &mut Response, name: &'static str, value: &str) {
+ let Some(value) = sanitize_routing_header_value(value) else {
+ return;
+ };
+ let Ok(value) = HeaderValue::from_str(&value) else {
+ return;
+ };
+ response
+ .headers_mut()
+ .insert(HeaderName::from_static(name), value);
}
fn sanitize_routing_header_value(value: &str) -> Option<String> {
@@ -366,112 +365,83 @@ fn sanitize_routing_header_value(value: &str) -> Option<String> {
(!value.is_empty()).then(|| value.chars().take(MAX_ROUTING_HEADER_VALUE_LEN).collect())
}
-fn metadata_from_headers(headers: &HeaderMap, inbound_format: ChatRequestType) -> RequestMetadata {
- RequestMetadata {
- request_id: request_id_from_headers(headers),
- inbound_format: Some(inbound_format),
- headers: normalized_headers(headers),
- }
-}
-
-fn request_id_from_headers(headers: &HeaderMap) -> Option<RequestId> {
- headers
- .get("x-request-id")
- .and_then(|value| value.to_str().ok())
- .and_then(|value| RequestId::new(value.to_string()).ok())
-}
-
-fn normalized_headers(headers: &HeaderMap) -> BTreeMap<String, Vec<String>> {
- let mut normalized = BTreeMap::<String, Vec<String>>::new();
- for (name, value) in headers {
- let Ok(value) = value.to_str() else {
- continue;
- };
- normalized
- .entry(name.as_str().to_ascii_lowercase())
- .or_default()
- .push(value.to_string());
- }
- normalized
-}
-
-fn llm_error(error: SwitchyardError) -> Response {
+fn algorithm_error(error: BoxError) -> Response {
+ let Some(error) = error.downcast_ref::<LlmClientError>() else {
+ return server_error(error.to_string());
+ };
match error {
- SwitchyardError::ModelNotFound { model } => (
- StatusCode::NOT_FOUND,
- Json(json!({
- "error": {
- "message": format!("No route registered for model {}", model.as_str()),
- "type": "model_not_found",
- "code": "model_not_found",
- }
- })),
- )
- .into_response(),
- SwitchyardError::InvalidConfig(message) | SwitchyardError::InvalidRequest(message) => (
+ LlmClientError::UnknownModel(model) => error_response(
+ StatusCode::BAD_GATEWAY,
+ format!("No upstream backend configured for model {model}"),
+ "upstream_error",
+ "upstream_model_not_found",
+ ),
+ LlmClientError::UnknownModelFormat { model, format } => error_response(
+ StatusCode::BAD_GATEWAY,
+ format!("No upstream backend configured for model {model} and format {format}"),
+ "upstream_error",
+ "upstream_format_not_found",
+ ),
+ LlmClientError::MissingModel => error_response(
StatusCode::BAD_REQUEST,
- Json(json!({
- "error": {
- "message": message,
- "type": "invalid_request_error",
- "code": "invalid_request_error",
- }
- })),
- )
- .into_response(),
- SwitchyardError::InvalidId(error) => (
+ error.to_string(),
+ "invalid_request_error",
+ "invalid_request_error",
+ ),
+ LlmClientError::ContextWindowExceeded { message, .. } => error_response(
StatusCode::BAD_REQUEST,
- Json(json!({
- "error": {
- "message": error.to_string(),
- "type": "invalid_request_error",
- "code": "invalid_request_error",
- }
- })),
- )
- .into_response(),
- SwitchyardError::UpstreamHttp {
- provider,
- status_code,
+ message,
+ "invalid_request_error",
+ "context_length_exceeded",
+ ),
+ LlmClientError::UpstreamHttp { status, body } => error_response(
+ StatusCode::from_u16(*status).unwrap_or(StatusCode::BAD_GATEWAY),
body,
- } => (
- StatusCode::from_u16(status_code).unwrap_or(StatusCode::BAD_GATEWAY),
- Json(json!({
- "error": {
- "message": body,
- "type": "upstream_error",
- "code": "upstream_error",
- "provider": provider,
- }
- })),
- )
- .into_response(),
- error => server_error(error.to_string()),
+ "upstream_error",
+ "upstream_error",
+ ),
+ LlmClientError::Translation(message)
+ | LlmClientError::Transport(message)
+ | LlmClientError::Stream(message) => error_response(
+ StatusCode::BAD_GATEWAY,
+ message,
+ "upstream_error",
+ "upstream_error",
+ ),
}
}
-fn server_error(message: String) -> Response {
- (
+fn server_error(message: impl Into<String>) -> Response {
+ error_response(
StatusCode::INTERNAL_SERVER_ERROR,
- Json(json!({
- "error": {
- "message": message,
- "type": "server_error",
- "code": "server_error",
- }
- })),
+ message,
+ "server_error",
+ "server_error",
)
- .into_response()
}
fn invalid_body_error(message: impl Into<String>) -> Response {
- (
+ error_response(
StatusCode::BAD_REQUEST,
+ message,
+ "invalid_request_error",
+ "invalid_body",
+ )
+}
+
+fn error_response(
+ status: StatusCode,
+ message: impl Into<String>,
+ error_type: &'static str,
+ code: &'static str,
+) -> Response {
+ (
+ status,
Json(json!({
"error": {
"message": message.into(),
- "type": "invalid_request_error",
- "code": "invalid_body",
+ "type": error_type,
+ "code": code,
}
})),
)
@@ -479,22 +449,7 @@ fn invalid_body_error(message: impl Into<String>) -> Response {
}
async fn models(State(state): State<ServerState>) -> Json<Value> {
- let entries = state.registry().served_models();
- Json(model_list_payload(&entries))
-}
-
-async fn stats() -> Response {
- match profile_stats_accumulator().snapshot() {
- Ok(snapshot) => Json(json!(snapshot)).into_response(),
- Err(error) => server_error(error.to_string()),
- }
-}
-
-async fn reset_stats() -> Response {
- match profile_stats_accumulator().reset() {
- Ok(()) => Json(json!({"status": "reset"})).into_response(),
- Err(error) => server_error(error.to_string()),
- }
+ Json(model_list_payload(state.models()))
}
async fn health() -> Json<Value> {
@@ -502,27 +457,21 @@ async fn health() -> Json<Value> {
}
async fn not_found() -> Response {
- (
+ error_response(
StatusCode::NOT_FOUND,
- Json(json!({
- "detail": "Not Found",
- })),
+ "Not Found",
+ "not_found",
+ "endpoint_not_found",
)
- .into_response()
}
-fn model_list_payload(entries: &[ServedModel]) -> Value {
- let data = entries.iter().map(model_entry_json).collect::<Vec<_>>();
- let model_ids = entries
- .iter()
- .map(|entry| entry.id.as_str().to_string())
- .collect::<Vec<_>>();
+fn model_list_payload<'a>(models: impl IntoIterator<Item = &'a str>) -> Value {
+ let model_ids = models.into_iter().map(str::to_string).collect::<Vec<_>>();
let first_id = model_ids.first().cloned();
let last_id = model_ids.last().cloned();
-
json!({
"object": "list",
- "data": data,
+ "data": model_ids.iter().map(|model| model_entry_json(model)).collect::<Vec<_>>(),
"first_id": first_id,
"last_id": last_id,
"has_more": false,
@@ -531,14 +480,14 @@ fn model_list_payload(entries: &[ServedModel]) -> Value {
})
}
-fn model_entry_json(entry: &ServedModel) -> Value {
+fn model_entry_json(model: &str) -> Value {
json!({
- "id": entry.id.as_str(),
+ "id": model,
"object": "model",
"type": "model",
"created": 0,
"owned_by": "switchyard",
- "display_name": entry.display_name,
+ "display_name": model,
"capabilities": {
"streaming": true,
"tool_calling": null,
@@ -552,115 +501,26 @@ fn model_entry_json(entry: &ServedModel) -> Value {
})
}
-fn startup_banner(options: &ServerRunOptions, registry: &ProfileRegistry) -> String {
- let entries = registry.served_models();
+fn startup_banner(options: &ServerRunOptions, state: &ServerState) -> String {
let scheme = if options.is_tls() { "https" } else { "http" };
- let listen_url = url_for_addr(scheme, options.addr);
- let local_url = local_url_for_addr(scheme, options.addr);
- let mut output = String::new();
-
- push_line(&mut output, "Switchyard Rust profile server");
- push_line(
- &mut output,
- format!(" config: {}", options.config.display()),
- );
- push_line(&mut output, format!(" listening: {listen_url}"));
- if local_url != listen_url {
- push_line(&mut output, format!(" local: {local_url}"));
- }
- push_line(&mut output, "");
- push_line(&mut output, " endpoints:");
- push_line(&mut output, " GET /health");
- push_line(&mut output, " GET /v1/models");
- push_line(&mut output, " POST /v1/chat/completions");
- push_line(&mut output, " POST /v1/messages");
- push_line(&mut output, " POST /v1/responses");
- push_line(&mut output, " GET /v1/routing/stats");
- push_line(&mut output, " POST /v1/routing/stats/reset");
- push_line(&mut output, "");
- push_line(&mut output, " available models:");
- for entry in &entries {
- if entry.display_name == entry.id.as_str() {
- push_line(&mut output, format!(" - {}", entry.id.as_str()));
- } else {
- push_line(
- &mut output,
- format!(" - {} ({})", entry.id.as_str(), entry.display_name),
- );
- }
- }
- push_line(&mut output, "");
- push_line(&mut output, " try:");
- push_line(&mut output, format!(" curl -s {local_url}/health"));
- push_line(&mut output, format!(" curl -s {local_url}/v1/models"));
- if let Some(entry) = entries.first() {
- let chat_payload = json!({
- "model": entry.id.as_str(),
- "messages": [{"role": "user", "content": "Say OK"}],
- "max_tokens": 256, // Space for thinking tokens
- });
- push_line(
- &mut output,
- format!(" curl -s {local_url}/v1/chat/completions -H 'content-type: application/json' -d '{chat_payload}'")
- );
- push_line(
- &mut output,
- format!(" curl -s {local_url}/v1/messages -H 'content-type: application/json' -H 'anthropic-version: 2023-06-01' -d '{chat_payload}'")
- );
-
- let responses_payload = json!({
- "model": entry.id.as_str(),
- "max_output_tokens": 256,
- "input": "Say OK",
- });
- push_line(&mut output, format!(" curl -s {local_url}/v1/responses -H 'content-type: application/json' -d '{responses_payload}'"));
- }
- if options.is_tls() {
- push_line(
- &mut output,
- " For self-signed certificates use `curl --insecure ..`",
- );
- }
- push_line(&mut output, "");
- push_line(&mut output, " stop: Ctrl-C");
- output
-}
-
-fn dry_run_summary(path: &Path, registry: &ProfileRegistry) -> String {
- let entries = registry.served_models();
- let mut output = String::new();
- push_line(
- &mut output,
- format!(
- "config OK: {}, public_models={}",
- path.display(),
- entries.len()
- ),
- );
- for entry in &entries {
- push_line(&mut output, format!(" - {}", entry.id.as_str()));
- }
- output
+ format!(
+ "Switchyard libsy server\n listening: {}\n routes: {}",
+ url_for_addr(scheme, options.addr),
+ state.models().collect::<Vec<_>>().join(", ")
+ )
}
-fn push_line(output: &mut String, line: impl AsRef<str>) {
- output.push_str(line.as_ref());
- output.push('\n');
+fn dry_run_summary(state: &ServerState) -> String {
+ format!(
+ "server OK: {}",
+ state.models().collect::<Vec<_>>().join(", ")
+ )
}
fn url_for_addr(scheme: &'static str, addr: SocketAddr) -> String {
format!("{scheme}://{}:{}", host_for_url(addr.ip()), addr.port())
}
-fn local_url_for_addr(scheme: &'static str, addr: SocketAddr) -> String {
- let host = match addr.ip() {
- std::net::IpAddr::V4(ip) if ip.is_unspecified() => "127.0.0.1".to_string(),
- std::net::IpAddr::V6(ip) if ip.is_unspecified() => "[::1]".to_string(),
- ip => host_for_url(ip),
- };
- format!("{scheme}://{host}:{}", addr.port())
-}
-
fn host_for_url(ip: std::net::IpAddr) -> String {
match ip {
std::net::IpAddr::V4(ip) => ip.to_string(),
diff --git a/crates/switchyard-server/src/registry.rs b/crates/switchyard-server/src/registry.rs
deleted file mode 100644
index 20b829c05..000000000
--- a/crates/switchyard-server/src/registry.rs
+++ /dev/null
@@ -1,119 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Profile registry used by the components-v2 Rust server.
-
-use std::sync::Arc;
-
-use serde::{Deserialize, Serialize};
-use switchyard_components_v2::{
- PassthroughProfileConfig, Profile, ProfileConfig, ProfileConfigPlan,
-};
-use switchyard_core::{ModelId, Result, SwitchyardError};
-
-/// Public model entry advertised by `/v1/models`.
-#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
-pub struct ServedModel {
- /// Public model or route ID accepted in inbound request bodies.
- pub id: ModelId,
- /// Human-readable label shown by CLI startup logs and `/v1/models`.
- pub display_name: String,
-}
-
-#[derive(Clone)]
-struct RegistryEntry {
- model: ServedModel,
- profile: Arc<dyn Profile>,
-}
-
-/// Exact-match registry from inbound `model` values to components-v2 profiles.
-#[derive(Clone, Default)]
-pub struct ProfileRegistry {
- entries: Vec<RegistryEntry>,
-}
-
-impl ProfileRegistry {
- /// Builds a registry from already-built exact-match profile runtimes.
- pub fn from_profiles(
- entries: impl IntoIterator<Item = (ModelId, Arc<dyn Profile>, String)>,
- ) -> Result<Self> {
- let mut registry = Self::default();
- for (model_id, profile, display_name) in entries {
- registry.insert(model_id, profile, display_name)?;
- }
- Ok(registry)
- }
-
- /// Builds a registry from a resolved profile config plan.
- pub fn from_plan(plan: &ProfileConfigPlan) -> Result<Self> {
- let mut registry = Self::default();
-
- for profile_id in plan.profile_ids() {
- let model_id = ModelId::new(profile_id.as_str())?;
- let profile: Arc<dyn Profile> = Arc::from(plan.build_profile(profile_id)?);
- let display_name = plan.profile_type(profile_id).unwrap_or("profile");
- registry.insert(model_id, profile, display_name)?;
- }
-
- for (_target_id, target) in plan.targets() {
- let profile: Arc<dyn Profile> = Arc::from(
- PassthroughProfileConfig {
- target: target.clone(),
- }
- .build_boxed()?,
- );
- registry.insert(
- ModelId::new(target.id.as_str())?,
- Arc::clone(&profile),
- target.model.as_str(),
- )?;
- }
-
- Ok(registry)
- }
-
- /// Returns the profile for an inbound request model.
- pub fn lookup(&self, model: Option<&str>) -> Result<Arc<dyn Profile>> {
- let Some(model) = model else {
- return Err(SwitchyardError::InvalidRequest(
- "request body must include a non-empty string `model`".to_string(),
- ));
- };
- let model = ModelId::new(model)?;
- self.entries
- .iter()
- .find(|entry| entry.model.id == model)
- .map(|entry| Arc::clone(&entry.profile))
- .ok_or(SwitchyardError::ModelNotFound { model })
- }
-
- /// Returns model entries in deterministic registration order.
- pub fn served_models(&self) -> Vec<ServedModel> {
- self.entries
- .iter()
- .map(|entry| entry.model.clone())
- .collect()
- }
-
- fn insert(
- &mut self,
- model_id: ModelId,
- profile: Arc<dyn Profile>,
- display_name: impl Into<String>,
- ) -> Result<()> {
- if self.entries.iter().any(|entry| entry.model.id == model_id) {
- return Err(SwitchyardError::DuplicateRegistration {
- kind: "model",
- id: model_id.to_string(),
- });
- }
- self.entries.push(RegistryEntry {
- model: ServedModel {
- id: model_id,
- display_name: display_name.into(),
- },
- profile,
- });
- Ok(())
- }
-}
diff --git a/crates/switchyard-server/src/response.rs b/crates/switchyard-server/src/response.rs
index 0a27718e3..9488d1764 100644
--- a/crates/switchyard-server/src/response.rs
+++ b/crates/switchyard-server/src/response.rs
@@ -1,144 +1,36 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! Response translation glue for Rust server endpoints.
+//! Response encoding glue for libsy server endpoints.
-use std::sync::Arc;
+use std::error::Error;
-use axum::response::sse::Sse;
-use serde_json::Value;
-use switchyard_core::{BoxResponseStream, ChatResponse, Result, StreamEvent, SwitchyardError};
-use switchyard_translation::{
- StreamTranslationState, TranslationEngine, TranslationPolicy, WireFormat,
-};
+use axum::response::{IntoResponse, Response as HttpResponse};
+use axum::Json;
+use libsy::{LlmResponse, Response as AlgorithmResponse};
+use switchyard_translation::{encode_aggregated_response, encode_stream, WireFormat};
-use crate::sse::{frame_stream, SseFrameStream};
+use crate::sse::frame_stream;
-pub(crate) enum TranslatedResponse {
- /// Complete JSON response body ready for an Axum JSON response.
- Buffered(Value),
- /// Framed SSE response ready for Axum streaming.
- Stream(Sse<SseFrameStream>),
-}
+type BoxError = Box<dyn Error + Send + Sync>;
-/// Translates a profile response into the endpoint's expected response shape.
-pub(crate) fn translate_chain_response(
- response: ChatResponse,
+/// Encodes a libsy response into the endpoint's wire format.
+pub(crate) fn into_http_response(
+ response: AlgorithmResponse,
target_format: WireFormat,
- translation: Arc<TranslationEngine>,
- policy: TranslationPolicy,
-) -> Result<TranslatedResponse> {
- match response {
- ChatResponse::OpenAiCompletion(response) => translate_buffered_response(
- response.into_body(),
- WireFormat::OpenAiChat,
- target_format,
- translation,
- &policy,
- ),
- ChatResponse::OpenAiResponsesCompletion(response) => translate_buffered_response(
- response.into_body(),
- WireFormat::OpenAiResponses,
+ requested_model: Option<String>,
+) -> Result<HttpResponse, BoxError> {
+ match response.llm_response {
+ LlmResponse::Agg(response) => Ok(Json(encode_aggregated_response(
+ &response,
target_format,
- translation,
- &policy,
- ),
- ChatResponse::AnthropicCompletion(response) => translate_buffered_response(
- response.into_body(),
- WireFormat::AnthropicMessages,
+ requested_model.as_deref(),
+ )?)
+ .into_response()),
+ LlmResponse::Stream(stream) => Ok(frame_stream(
+ encode_stream(stream, target_format, requested_model)?,
target_format,
- translation,
- &policy,
- ),
- ChatResponse::OpenAiStream(stream) => Ok(TranslatedResponse::Stream(
- translate_stream_response(stream, WireFormat::OpenAiChat, target_format, translation),
- )),
- ChatResponse::OpenAiResponsesStream(stream) => {
- Ok(TranslatedResponse::Stream(translate_stream_response(
- stream,
- WireFormat::OpenAiResponses,
- target_format,
- translation,
- )))
- }
- ChatResponse::AnthropicStream(stream) => {
- Ok(TranslatedResponse::Stream(translate_stream_response(
- stream,
- WireFormat::AnthropicMessages,
- target_format,
- translation,
- )))
- }
- }
-}
-
-fn translate_buffered_response(
- body: Value,
- source_format: WireFormat,
- target_format: WireFormat,
- translation: Arc<TranslationEngine>,
- policy: &TranslationPolicy,
-) -> Result<TranslatedResponse> {
- if source_format == target_format {
- return Ok(TranslatedResponse::Buffered(body));
+ )
+ .into_response()),
}
-
- let output = translation
- .translate_response(source_format, target_format, &body, policy)
- .map_err(|error| {
- SwitchyardError::Other(format!(
- "failed to translate {source_format} response to {target_format}: {error}"
- ))
- })?;
- Ok(TranslatedResponse::Buffered(output.body))
-}
-
-fn translate_stream_response(
- mut stream: BoxResponseStream,
- source_format: WireFormat,
- target_format: WireFormat,
- translation: Arc<TranslationEngine>,
-) -> Sse<SseFrameStream> {
- let events = async_stream::try_stream! {
- let mut translation_state = StreamTranslationState::new(source_format, target_format);
- while let Some(event) = futures_util::StreamExt::next(&mut stream).await {
- let event = event?;
- match event {
- StreamEvent::Json(value) => {
- if source_format == target_format {
- yield value;
- } else {
- for translated in translation.translate_event(
- &mut translation_state,
- source_format,
- target_format,
- &value,
- ).map_err(|error| {
- SwitchyardError::Other(format!(
- "failed to translate {source_format} stream event to {target_format}: {error}"
- ))
- })? {
- yield translated;
- }
- }
- }
- StreamEvent::Text(text) => {
- yield Value::String(text);
- }
- }
- }
-
- if source_format != target_format {
- for translated in translation.finish_stream(&mut translation_state, target_format)
- .map_err(|error| {
- SwitchyardError::Other(format!(
- "failed to finish {target_format} stream translation: {error}"
- ))
- })? {
- yield translated;
- }
- }
- };
-
- frame_stream(events, target_format)
}
diff --git a/crates/switchyard-server/src/sse.rs b/crates/switchyard-server/src/sse.rs
index 3d4f0e38d..d1c25ce4d 100644
--- a/crates/switchyard-server/src/sse.rs
+++ b/crates/switchyard-server/src/sse.rs
@@ -3,35 +3,47 @@
//! SSE framing helpers for OpenAI, Anthropic, and Responses endpoints.
+use std::convert::Infallible;
+
use axum::response::sse::{Event, Sse};
use futures_util::Stream;
use serde_json::{json, Value};
-use switchyard_core::{Result, SwitchyardError};
-use switchyard_translation::WireFormat;
+use switchyard_translation::{RawEventStream, WireFormat};
/// Boxed stream type accepted by Axum's SSE response wrapper.
pub(crate) type SseFrameStream =
- std::pin::Pin<Box<dyn Stream<Item = std::result::Result<Event, SwitchyardError>> + Send>>;
+ std::pin::Pin<Box<dyn Stream<Item = Result<Event, Infallible>> + Send>>;
/// Converts translated JSON events into endpoint-specific SSE frames.
pub(crate) fn frame_stream(
- stream: impl Stream<Item = Result<Value>> + Send + 'static,
+ stream: RawEventStream,
target_format: WireFormat,
) -> Sse<SseFrameStream> {
let framed = async_stream::stream! {
- let mut stream = Box::pin(stream);
+ let mut stream = stream;
+ let mut failed = false;
while let Some(item) = futures_util::StreamExt::next(&mut stream).await {
- match item {
- Ok(value) => yield frame_event(target_format, value),
+ let event = match item {
+ Ok(value) => match frame_event(target_format, value) {
+ Ok(event) => event,
+ Err(error) => {
+ failed = true;
+ error_event(target_format, error.to_string())
+ }
+ },
Err(error) => {
tracing::warn!(error = %error, "stream iteration failed");
- yield Ok(error_event(target_format, error.to_string()));
- return;
+ failed = true;
+ error_event(target_format, error.to_string())
}
+ };
+ yield Ok(event);
+ if failed {
+ break;
}
}
- if target_format == WireFormat::OpenAiChat {
+ if !failed && target_format == WireFormat::OpenAiChat {
yield Ok(Event::default().data("[DONE]"));
}
};
@@ -39,24 +51,16 @@ pub(crate) fn frame_stream(
Sse::new(Box::pin(framed) as SseFrameStream)
}
-fn frame_event(
- target_format: WireFormat,
- value: Value,
-) -> std::result::Result<Event, SwitchyardError> {
+fn frame_event(target_format: WireFormat, value: Value) -> Result<Event, axum::Error> {
match target_format {
- WireFormat::OpenAiChat => Event::default()
- .json_data(value)
- .map_err(|error| SwitchyardError::Other(error.to_string())),
+ WireFormat::OpenAiChat => Event::default().json_data(value),
WireFormat::AnthropicMessages | WireFormat::OpenAiResponses => {
let event_type = value
.get("type")
.and_then(Value::as_str)
.unwrap_or("message")
.to_string();
- Event::default()
- .event(event_type)
- .json_data(value)
- .map_err(|error| SwitchyardError::Other(error.to_string()))
+ Event::default().event(event_type).json_data(value)
}
}
}
@@ -86,3 +90,35 @@ fn error_event(target_format: WireFormat, message: String) -> Event {
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use std::{error::Error, io};
+
+ use axum::{body::to_bytes, response::IntoResponse};
+ use futures_util::stream;
+
+ use super::*;
+
+ type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
+
+ #[tokio::test]
+ async fn stream_error_terminates_without_done_marker() -> TestResult {
+ let failure: Box<dyn Error + Send + Sync> = Box::new(io::Error::other("boom"));
+ let stream: RawEventStream = Box::pin(stream::iter(vec![
+ Ok(json!({"id": "before"})),
+ Err(failure),
+ Ok(json!({"id": "after"})),
+ ]));
+
+ let response = frame_stream(stream, WireFormat::OpenAiChat).into_response();
+ let body = String::from_utf8(to_bytes(response.into_body(), usize::MAX).await?.to_vec())?;
+
+ // A stream error is terminal: later chunks and success markers must not be emitted.
+ assert!(body.contains("before"));
+ assert!(body.contains("boom"));
+ assert!(!body.contains("after"));
+ assert!(!body.contains("[DONE]"));
+ Ok(())
+ }
+}
diff --git a/crates/switchyard-server/tests/profile_migration.rs b/crates/switchyard-server/tests/profile_migration.rs
deleted file mode 100644
index abcb5cd35..000000000
--- a/crates/switchyard-server/tests/profile_migration.rs
+++ /dev/null
@@ -1,837 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Migration tests proving components-v2 profiles cover legacy serving contracts.
-
-use std::io::ErrorKind;
-use std::io::{Read, Write};
-use std::net::{SocketAddr, TcpListener as StdTcpListener, TcpStream};
-use std::sync::{Arc, Mutex};
-use std::thread;
-
-use axum::body::Body;
-use axum::http::{Request, StatusCode};
-use http_body_util::BodyExt;
-use serde_json::{json, Value};
-use switchyard_components_v2::{
- parse_profile_config_str, profile_stats_accumulator, ProfileConfigFormat,
-};
-use switchyard_server::{build_switchyard_router, ServerState};
-use tokio::sync::Mutex as TokioMutex;
-use tower::ServiceExt;
-
-static STATS_TEST_LOCK: TokioMutex<()> = TokioMutex::const_new(());
-
-#[tokio::test]
-async fn noop_profile_serves_all_inbound_formats_without_upstream() -> TestResult {
- let _stats_guard = STATS_TEST_LOCK.lock().await;
- reset_stats()?;
- let app = build_switchyard_router(state_from_yaml(
- r#"
-profiles:
- bench:
- type: noop
-"#,
- )?);
-
- let models = app
- .clone()
- .oneshot(request("GET", "/v1/models", None)?)
- .await?;
- assert_eq!(models.status(), StatusCode::OK);
- assert_eq!(json_body(models).await?["model_pool"], json!(["bench"]));
-
- let chat = app
- .clone()
- .oneshot(request(
- "POST",
- "/v1/chat/completions",
- Some(chat_body("bench")),
- )?)
- .await?;
- assert_eq!(chat.status(), StatusCode::OK);
- assert_eq!(
- json_body(chat).await?["choices"][0]["message"]["content"],
- "ok"
- );
-
- let messages = app
- .clone()
- .oneshot(request(
- "POST",
- "/v1/messages",
- Some(json!({
- "model": "bench",
- "max_tokens": 16,
- "messages": [{"role": "user", "content": "hi"}],
- })),
- )?)
- .await?;
- assert_eq!(messages.status(), StatusCode::OK);
- assert_eq!(json_body(messages).await?["content"][0]["text"], "ok");
-
- let responses = app
- .oneshot(request(
- "POST",
- "/v1/responses",
- Some(json!({"model": "bench", "input": "hi"})),
- )?)
- .await?;
- assert_eq!(responses.status(), StatusCode::OK);
- assert_eq!(
- json_body(responses).await?["output"][0]["content"][0]["text"],
- "ok"
- );
- Ok(())
-}
-
-#[tokio::test]
-async fn passthrough_profile_routes_all_inbound_formats_to_configured_target() -> TestResult {
- let _stats_guard = STATS_TEST_LOCK.lock().await;
- reset_stats()?;
- let Some(stub) = HttpStub::start(vec![
- StubResponse::ok(),
- StubResponse::ok(),
- StubResponse::ok(),
- ])?
- else {
- log_loopback_bind_skip();
- return Ok(());
- };
- let app = build_switchyard_router(state_from_yaml(&format!(
- r#"
-targets:
- direct:
- model: provider/direct
- format: openai
- base_url: {base_url}
-profiles:
- direct-profile:
- type: passthrough
- target: direct
-"#,
- base_url = stub.base_url
- ))?);
-
- let chat = app
- .clone()
- .oneshot(request(
- "POST",
- "/v1/chat/completions",
- Some(chat_body("direct-profile")),
- )?)
- .await?;
- assert_eq!(chat.status(), StatusCode::OK);
- let body = json_body(chat).await?;
- assert_eq!(body["choices"][0]["message"]["content"], "stub-ok");
-
- let messages = app
- .clone()
- .oneshot(request(
- "POST",
- "/v1/messages",
- Some(json!({
- "model": "direct-profile",
- "max_tokens": 16,
- "messages": [{"role": "user", "content": "hi"}],
- })),
- )?)
- .await?;
- assert_eq!(messages.status(), StatusCode::OK);
- assert_eq!(json_body(messages).await?["content"][0]["text"], "stub-ok");
-
- let responses = app
- .oneshot(request(
- "POST",
- "/v1/responses",
- Some(json!({"model": "direct-profile", "input": "hi"})),
- )?)
- .await?;
- assert_eq!(responses.status(), StatusCode::OK);
- assert_eq!(
- json_body(responses).await?["output"][0]["content"][0]["text"],
- "stub-ok"
- );
-
- let seen = stub.requests()?;
- assert_eq!(seen.len(), 3);
- for request in seen {
- assert_eq!(request["method"], "POST");
- assert_eq!(request["path"], "/v1/chat/completions");
- assert_eq!(request["body"]["model"], "provider/direct");
- }
- Ok(())
-}
-
-#[tokio::test]
-async fn random_routing_profile_covers_strong_and_weak_selection() -> TestResult {
- let _stats_guard = STATS_TEST_LOCK.lock().await;
- reset_stats()?;
- let cases = [(1.0, "provider/strong"), (0.0, "provider/weak")];
- for (strong_probability, expected_model) in cases {
- let Some(stub) = HttpStub::start(vec![StubResponse::ok()])? else {
- log_loopback_bind_skip();
- return Ok(());
- };
- let app = build_switchyard_router(state_from_yaml(&format!(
- r#"
-targets:
- strong:
- model: provider/strong
- format: openai
- base_url: {base_url}
- weak:
- model: provider/weak
- format: openai
- base_url: {base_url}
-profiles:
- random-profile:
- type: random-routing
- strong: strong
- weak: weak
- strong_probability: {strong_probability}
- rng_seed: 7
-"#,
- base_url = stub.base_url
- ))?);
-
- let response = app
- .oneshot(request(
- "POST",
- "/v1/chat/completions",
- Some(chat_body("random-profile")),
- )?)
- .await?;
-
- assert_eq!(response.status(), StatusCode::OK);
- let seen = stub.requests()?;
- assert_eq!(seen.len(), 1);
- assert_eq!(seen[0]["body"]["model"], expected_model);
- }
- Ok(())
-}
-
-#[tokio::test]
-async fn stage_router_profile_threshold_zero_uses_dimensions_signal_path() -> TestResult {
- let _stats_guard = STATS_TEST_LOCK.lock().await;
- reset_stats()?;
- let Some(stub) = HttpStub::start(vec![StubResponse::ok()])? else {
- log_loopback_bind_skip();
- return Ok(());
- };
- let app = build_switchyard_router(state_from_yaml(&format!(
- r#"
-targets:
- strong:
- model: provider/strong
- format: openai
- base_url: {base_url}
- weak:
- model: provider/weak
- format: openai
- base_url: {base_url}
-profiles:
- smart-stage_router:
- type: stage_router
- capable: strong
- efficient: weak
- fallback_target_on_evict: strong
- picker: capable_first
- confidence_threshold: 0.0
-"#,
- base_url = stub.base_url
- ))?);
-
- let response = app
- .clone()
- .oneshot(request(
- "POST",
- "/v1/chat/completions",
- Some(chat_body("smart-stage_router")),
- )?)
- .await?;
- assert_eq!(response.status(), StatusCode::OK);
-
- let seen = stub.requests()?;
- assert_eq!(seen.len(), 1);
- assert_eq!(seen[0]["body"]["model"], "provider/weak");
-
- let stats = app.oneshot(request("GET", "/v1/stats", None)?).await?;
- let stats = json_body(stats).await?;
- assert_eq!(stats["routing_decisions"]["stage_router"]["dimensions"], 1);
- assert_eq!(stats["classifier"]["total_requests"], 0);
- assert_eq!(stats["models"]["provider/weak"]["tier"], "efficient");
- Ok(())
-}
-
-#[tokio::test]
-async fn stage_router_profile_threshold_one_uses_llm_classifier_path() -> TestResult {
- let _stats_guard = STATS_TEST_LOCK.lock().await;
- reset_stats()?;
- let Some(stub) = HttpStub::start(vec![
- StubResponse::classifier("capable"),
- StubResponse::ok(),
- ])?
- else {
- log_loopback_bind_skip();
- return Ok(());
- };
- let app = build_switchyard_router(state_from_yaml(&format!(
- r#"
-targets:
- strong:
- model: provider/strong
- format: openai
- base_url: {base_url}
- weak:
- model: provider/weak
- format: openai
- base_url: {base_url}
-profiles:
- smart-stage_router:
- type: stage_router
- capable: strong
- efficient: weak
- fallback_target_on_evict: strong
- picker: capable_first
- confidence_threshold: 1.0
- classifier:
- model: classifier/model
- api_key: test-key
- base_url: {base_url}
- timeout_secs: 5.0
-"#,
- base_url = stub.base_url
- ))?);
-
- let response = app
- .clone()
- .oneshot(request(
- "POST",
- "/v1/chat/completions",
- Some(chat_body("smart-stage_router")),
- )?)
- .await?;
- assert_eq!(response.status(), StatusCode::OK);
-
- let seen = stub.requests()?;
- assert_eq!(seen.len(), 2);
- assert_eq!(seen[0]["body"]["model"], "classifier/model");
- assert_eq!(seen[1]["body"]["model"], "provider/strong");
-
- let stats = app.oneshot(request("GET", "/v1/stats", None)?).await?;
- let stats = json_body(stats).await?;
- assert_eq!(
- stats["routing_decisions"]["stage_router"]["llm-classifier"],
- 1
- );
- assert_eq!(stats["classifier"]["total_requests"], 1);
- assert_eq!(stats["models"]["provider/strong"]["tier"], "capable");
- Ok(())
-}
-
-#[tokio::test]
-async fn stage_router_profile_retries_fallback_after_context_overflow() -> TestResult {
- let _stats_guard = STATS_TEST_LOCK.lock().await;
- reset_stats()?;
- let Some(stub) = HttpStub::start(vec![StubResponse::context_overflow(), StubResponse::ok()])?
- else {
- log_loopback_bind_skip();
- return Ok(());
- };
- let app = build_switchyard_router(state_from_yaml(&format!(
- r#"
-targets:
- strong:
- model: provider/strong
- format: openai
- base_url: {base_url}
- weak:
- model: provider/weak
- format: openai
- base_url: {base_url}
-profiles:
- smart-stage_router:
- type: stage_router
- capable: strong
- efficient: weak
- fallback_target_on_evict: strong
- picker: efficient_first
- confidence_threshold: 0.7
-"#,
- base_url = stub.base_url
- ))?);
-
- let response = app
- .clone()
- .oneshot(request(
- "POST",
- "/v1/chat/completions",
- Some(chat_body("smart-stage_router")),
- )?)
- .await?;
- assert_eq!(response.status(), StatusCode::OK);
-
- let seen = stub.requests()?;
- assert_eq!(seen.len(), 2);
- assert_eq!(seen[0]["body"]["model"], "provider/weak");
- assert_eq!(seen[1]["body"]["model"], "provider/strong");
-
- let stats = app.oneshot(request("GET", "/v1/stats", None)?).await?;
- let stats = json_body(stats).await?;
- assert_eq!(stats["routing_decisions"]["stage_router"]["fall_open"], 1);
- assert_eq!(stats["models"]["provider/strong"]["tier"], "capable");
- Ok(())
-}
-
-#[tokio::test]
-async fn latency_service_profile_uses_health_selection_and_retries_failed_target() -> TestResult {
- let _stats_guard = STATS_TEST_LOCK.lock().await;
- reset_stats()?;
- let Some(stub) = HttpStub::start(vec![
- StubResponse::latency_health(),
- StubResponse::error(503, json!({"error": {"message": "fast unavailable"}})),
- StubResponse::ok(),
- ])?
- else {
- log_loopback_bind_skip();
- return Ok(());
- };
- let app = build_switchyard_router(state_from_yaml(&latency_profile_yaml(&stub.origin, 1))?);
-
- let response = app
- .oneshot(request(
- "POST",
- "/v1/chat/completions",
- Some(chat_body("latency-profile")),
- )?)
- .await?;
- assert_eq!(response.status(), StatusCode::OK);
- assert_eq!(
- json_body(response).await?["choices"][0]["message"]["content"],
- "stub-ok"
- );
-
- let seen = stub.requests()?;
- assert_eq!(seen.len(), 3);
- assert_eq!(seen[0]["method"], "GET");
- assert!(seen[0]["path"]
- .as_str()
- .unwrap_or_default()
- .starts_with("/v1/endpoints/health?"));
- assert!(seen[0]["path"]
- .as_str()
- .unwrap_or_default()
- .contains("endpoint_ids=fast"));
- assert!(seen[0]["path"]
- .as_str()
- .unwrap_or_default()
- .contains("endpoint_ids=slow"));
- assert_eq!(seen[1]["path"], "/fast/v1/chat/completions");
- assert_eq!(seen[1]["body"]["model"], "provider/fast");
- assert_eq!(seen[2]["path"], "/slow/v1/chat/completions");
- assert_eq!(seen[2]["body"]["model"], "provider/slow");
- Ok(())
-}
-
-#[tokio::test]
-async fn latency_service_profile_propagates_last_upstream_error_after_retries() -> TestResult {
- let _stats_guard = STATS_TEST_LOCK.lock().await;
- reset_stats()?;
- let Some(stub) = HttpStub::start(vec![
- StubResponse::latency_health(),
- StubResponse::error(503, json!({"error": {"message": "fast unavailable"}})),
- StubResponse::error(502, json!({"error": {"message": "slow unavailable"}})),
- ])?
- else {
- log_loopback_bind_skip();
- return Ok(());
- };
- let app = build_switchyard_router(state_from_yaml(&latency_profile_yaml(&stub.origin, 1))?);
-
- let response = app
- .oneshot(request(
- "POST",
- "/v1/chat/completions",
- Some(chat_body("latency-profile")),
- )?)
- .await?;
- assert_eq!(response.status(), StatusCode::BAD_GATEWAY);
- let body = json_body(response).await?;
- assert_eq!(body["error"]["type"], "upstream_error");
- assert!(body["error"]["message"]
- .as_str()
- .unwrap_or_default()
- .contains("slow unavailable"));
-
- let seen = stub.requests()?;
- assert_eq!(seen.len(), 3);
- assert_eq!(seen[1]["body"]["model"], "provider/fast");
- assert_eq!(seen[2]["body"]["model"], "provider/slow");
- Ok(())
-}
-
-#[tokio::test]
-async fn profile_config_negative_cases_fail_before_legacy_deletion() -> TestResult {
- let unknown_profile_type = state_from_yaml(
- r#"
-profiles:
- bad:
- type: not-a-profile
-"#,
- )
- .err()
- .ok_or("unknown profile type should fail")?
- .to_string();
- assert!(unknown_profile_type.contains("unknown profile type"));
- assert!(unknown_profile_type.contains("not-a-profile"));
-
- let unknown_target = state_from_yaml(
- r#"
-targets: {}
-profiles:
- bad:
- type: passthrough
- target: missing
-"#,
- )
- .err()
- .ok_or("unknown target should fail")?
- .to_string();
- assert!(unknown_target.contains("unknown target missing"));
-
- let invalid_config = state_from_yaml(
- r#"
-targets:
- strong:
- model: provider/strong
- format: openai
- weak:
- model: provider/weak
- format: openai
-profiles:
- random-profile:
- type: random-routing
- strong: strong
- weak: weak
- strong_probability: 2.0
-"#,
- )
- .err()
- .ok_or("invalid random-routing probability should fail")?
- .to_string();
- assert!(invalid_config.contains("strong_probability"));
- Ok(())
-}
-
-#[tokio::test]
-async fn unknown_inbound_model_stays_a_client_error() -> TestResult {
- let app = build_switchyard_router(state_from_yaml(
- r#"
-profiles:
- bench:
- type: noop
-"#,
- )?);
-
- let response = app
- .oneshot(request(
- "POST",
- "/v1/chat/completions",
- Some(chat_body("missing")),
- )?)
- .await?;
-
- assert_eq!(response.status(), StatusCode::NOT_FOUND);
- assert_eq!(
- json_body(response).await?["error"]["code"],
- "model_not_found"
- );
- Ok(())
-}
-
-#[derive(Clone)]
-struct StubResponse {
- status: u16,
- body: Value,
-}
-
-impl StubResponse {
- fn ok() -> Self {
- Self {
- status: 200,
- body: json!({
- "id": "chatcmpl-stub",
- "object": "chat.completion",
- "model": "stub",
- "choices": [{
- "index": 0,
- "message": {"role": "assistant", "content": "stub-ok"},
- "finish_reason": "stop",
- }],
- "usage": {
- "prompt_tokens": 1,
- "completion_tokens": 1,
- "total_tokens": 2,
- },
- }),
- }
- }
-
- fn latency_health() -> Self {
- Self {
- status: 200,
- body: json!({
- "endpoint_health": {
- "fast": {"status": "healthy", "last_latency_ms": 10.0},
- "slow": {"status": "degraded", "last_latency_ms": 1.0},
- },
- }),
- }
- }
-
- fn error(status: u16, body: Value) -> Self {
- Self { status, body }
- }
-
- fn classifier(tier: &str) -> Self {
- Self {
- status: 200,
- body: json!({
- "id": "chatcmpl-classifier",
- "object": "chat.completion",
- "model": "classifier/model",
- "choices": [{
- "index": 0,
- "message": {"role": "assistant", "content": json!({"tier": tier}).to_string()},
- "finish_reason": "stop",
- }],
- "usage": {
- "prompt_tokens": 3,
- "completion_tokens": 1,
- "total_tokens": 4,
- },
- }),
- }
- }
-
- fn context_overflow() -> Self {
- Self {
- status: 400,
- body: json!({
- "error": {
- "code": "context_length_exceeded",
- "message": "maximum context length exceeded",
- },
- }),
- }
- }
-}
-
-/// In-process HTTP stub that serves one response per accepted request.
-struct HttpStub {
- /// Server origin used by latency-service health polling.
- origin: String,
- /// OpenAI-compatible base URL ending in `/v1`.
- base_url: String,
- /// Bound address used to wake blocked accepts during drop.
- addr: SocketAddr,
- /// Number of requests the stub thread expects before exiting.
- expected_requests: usize,
- /// Captured method, path, and JSON body for each real request.
- requests: Arc<Mutex<Vec<Value>>>,
- /// Background accept loop joined on drop after wake-up connects.
- handle: Option<thread::JoinHandle<()>>,
-}
-
-impl HttpStub {
- /// Binds an ephemeral port and returns `None` when loopback is sandbox-denied.
- fn start(responses: Vec<StubResponse>) -> TestResult<Option<Self>> {
- let listener = match StdTcpListener::bind("127.0.0.1:0") {
- Ok(listener) => listener,
- Err(error) if error.kind() == ErrorKind::PermissionDenied => return Ok(None),
- Err(error) => return Err(error.into()),
- };
- let addr = listener.local_addr()?;
- let expected_requests = responses.len();
- let requests = Arc::new(Mutex::new(Vec::new()));
- let thread_requests = Arc::clone(&requests);
- let handle = thread::spawn(move || {
- for response in responses {
- let Ok((mut stream, _addr)) = listener.accept() else {
- return;
- };
- if let Ok((method, path, body)) = read_http_request(&mut stream) {
- if let Ok(mut requests) = thread_requests.lock() {
- requests.push(json!({"method": method, "path": path, "body": body}));
- }
- }
- let body = response.body.to_string();
- let reason = if response.status == 200 {
- "OK"
- } else {
- "ERROR"
- };
- let _ = write!(
- stream,
- "HTTP/1.1 {} {}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
- response.status,
- reason,
- body.len(),
- body
- );
- }
- });
-
- Ok(Some(Self {
- origin: format!("http://{addr}"),
- base_url: format!("http://{addr}/v1"),
- addr,
- expected_requests,
- requests,
- handle: Some(handle),
- }))
- }
-
- fn requests(&self) -> TestResult<Vec<Value>> {
- self.requests
- .lock()
- .map(|requests| requests.clone())
- .map_err(|_| "stub request mutex poisoned".into())
- }
-}
-
-impl Drop for HttpStub {
- fn drop(&mut self) {
- // Wake pending accepts before joining the stub thread.
- for _ in 0..self.expected_requests {
- let _ = TcpStream::connect(self.addr);
- }
- if let Some(handle) = self.handle.take() {
- let _ = handle.join();
- }
- }
-}
-
-/// Reads one small HTTP request, tolerating partial reads until headers and
-/// the declared `content-length` body are available.
-fn read_http_request(stream: &mut TcpStream) -> TestResult<(String, String, Value)> {
- let mut buffer = Vec::new();
- let mut header_end = None;
- while header_end.is_none() {
- let mut chunk = [0; 1024];
- let read = stream.read(&mut chunk)?;
- if read == 0 {
- break;
- }
- buffer.extend_from_slice(&chunk[..read]);
- header_end = find_bytes(&buffer, b"\r\n\r\n").map(|index| index + 4);
- }
-
- let Some(body_start) = header_end else {
- return Err("HTTP request headers were incomplete".into());
- };
- let headers = std::str::from_utf8(&buffer[..body_start])?;
- let mut request_line = headers
- .lines()
- .next()
- .ok_or("HTTP request line was missing")?
- .split_whitespace();
- let method = request_line
- .next()
- .ok_or("HTTP request line was missing a method")?
- .to_string();
- let path = request_line
- .next()
- .ok_or("HTTP request line was missing a path")?
- .to_string();
- let content_length = headers
- .lines()
- .find_map(|line| {
- let (name, value) = line.split_once(':')?;
- name.eq_ignore_ascii_case("content-length")
- .then(|| value.trim().parse::<usize>().ok())
- .flatten()
- })
- .unwrap_or(0);
-
- while buffer.len() < body_start + content_length {
- let mut chunk = [0; 1024];
- let read = stream.read(&mut chunk)?;
- if read == 0 {
- break;
- }
- buffer.extend_from_slice(&chunk[..read]);
- }
-
- if buffer.len() < body_start + content_length {
- return Err("HTTP request body was incomplete".into());
- }
- let body = if content_length == 0 {
- Value::Null
- } else {
- serde_json::from_slice(&buffer[body_start..body_start + content_length])?
- };
- Ok((method, path, body))
-}
-
-fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option<usize> {
- haystack
- .windows(needle.len())
- .position(|window| window == needle)
-}
-
-fn state_from_yaml(input: &str) -> TestResult<ServerState> {
- let plan = parse_profile_config_str(input, ProfileConfigFormat::Yaml)?.resolve()?;
- Ok(ServerState::from_plan(&plan)?)
-}
-
-fn chat_body(model: &str) -> Value {
- json!({"model": model, "messages": [{"role": "user", "content": "hi"}]})
-}
-
-fn latency_profile_yaml(origin: &str, max_retries: usize) -> String {
- format!(
- r#"
-targets:
- fast:
- model: provider/fast
- format: openai
- base_url: {origin}/fast/v1
- slow:
- model: provider/slow
- format: openai
- base_url: {origin}/slow/v1
-profiles:
- latency-profile:
- type: latency-service
- latency_service_url: {origin}
- targets: [fast, slow]
- max_retries: {max_retries}
-"#
- )
-}
-
-fn request(method: &str, uri: &str, body: Option<Value>) -> TestResult<Request<Body>> {
- let builder = Request::builder()
- .method(method)
- .uri(uri)
- .header("content-type", "application/json");
- let body = body.map_or_else(Body::empty, |body| Body::from(body.to_string()));
- Ok(builder.body(body)?)
-}
-
-async fn json_body(response: axum::response::Response) -> TestResult<Value> {
- let bytes = response.into_body().collect().await?.to_bytes();
- Ok(serde_json::from_slice(&bytes)?)
-}
-
-fn reset_stats() -> switchyard_core::Result<()> {
- profile_stats_accumulator().reset()
-}
-
-fn log_loopback_bind_skip() {
- eprintln!("SKIP: permission denied binding loopback socket");
-}
-
-type TestResult<T = ()> = std::result::Result<T, Box<dyn std::error::Error + Send + Sync>>;
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 90ee30b1b..0db025d20 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -1,841 +1,359 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! Integration tests for the components-v2 Rust profile server.
-
-use std::io::ErrorKind;
-use std::io::{Read, Write};
-use std::net::{SocketAddr, TcpListener as StdTcpListener, TcpStream};
-use std::sync::{Arc, Mutex, OnceLock};
-use std::thread;
-
-use async_trait::async_trait;
-use axum::body::Body;
-use axum::http::{Request, StatusCode};
+//! Integration tests for the libsy Rust server.
+
+use std::collections::{BTreeMap, HashSet};
+use std::convert::Infallible;
+use std::error::Error;
+use std::sync::Arc;
+
+use axum::body::{Body, Bytes};
+use axum::extract::State;
+use axum::http::{Request as HttpRequest, StatusCode};
+use axum::response::sse::{Event, Sse};
+use axum::response::{IntoResponse, Response as HttpResponse};
+use axum::routing::post;
+use axum::{Json, Router};
use http_body_util::BodyExt;
+use libsy::algorithms::Random;
+use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient};
use serde_json::{json, Value};
-use switchyard_components_v2::{
- parse_profile_config_str, profile_stats_accumulator, Profile, ProfileConfigFormat,
- ProfileInput, ProfileResponse, RoutingMetadata,
-};
-use switchyard_core::{
- ChatRequestType, ChatResponse, ModelId, Result, StreamEvent, SwitchyardError,
-};
-use switchyard_server::{build_switchyard_router, ProfileRegistry, ServerState};
+use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
+use switchyard_server::{build_switchyard_router, ServerState};
+use tokio::net::TcpListener;
+use tokio::sync::Mutex;
+use tokio::task::JoinHandle;
use tower::ServiceExt;
-#[tokio::test]
-async fn minimal_noop_config_boots_and_serves_core_routes() -> TestResult {
- let _stats_guard = stats_guard().await;
- reset_stats()?;
- let app = build_switchyard_router(state_from_yaml(
- r#"
-profiles:
- bench:
- type: noop
-"#,
- )?);
-
- let health = app
- .clone()
- .oneshot(request("GET", "/health", None)?)
- .await?;
- assert_eq!(health.status(), StatusCode::OK);
- assert_eq!(json_body(health).await?, json!({"status": "ok"}));
+type TestError = Box<dyn Error + Send + Sync>;
+type TestResult<T = ()> = Result<T, TestError>;
- let models = app
- .clone()
- .oneshot(request("GET", "/v1/models", None)?)
- .await?;
- assert_eq!(models.status(), StatusCode::OK);
- let models = json_body(models).await?;
- assert_eq!(models["object"], "list");
- assert_eq!(models["data"][0]["id"], "bench");
- assert_eq!(models["default_model"], "bench");
- assert_eq!(models["model_pool"], json!(["bench"]));
-
- let chat = app
- .oneshot(request(
- "POST",
- "/v1/chat/completions",
- Some(json!({
- "model": "bench",
- "messages": [{"role": "user", "content": "hi"}],
- })),
- )?)
- .await?;
- assert_eq!(chat.status(), StatusCode::OK);
- assert_eq!(
- json_body(chat).await?["choices"][0]["message"]["content"],
- "ok"
- );
- Ok(())
-}
+const ROUTE_MODEL: &str = "switchyard/random";
-#[tokio::test]
-async fn all_request_endpoints_route_through_selected_profile() -> TestResult {
- let _stats_guard = stats_guard().await;
- reset_stats()?;
- let app = build_switchyard_router(state_from_yaml(
- r#"
-profiles:
- bench:
- type: noop
-"#,
- )?);
-
- let anthropic = app
- .clone()
- .oneshot(request(
- "POST",
- "/v1/messages",
- Some(json!({
- "model": "bench",
- "max_tokens": 16,
- "messages": [{"role": "user", "content": "hi"}],
- })),
- )?)
- .await?;
- assert_eq!(anthropic.status(), StatusCode::OK);
- let anthropic = json_body(anthropic).await?;
- assert_eq!(anthropic["type"], "message");
- assert_eq!(anthropic["content"][0]["text"], "ok");
-
- let responses = app
- .oneshot(request(
- "POST",
- "/v1/responses",
- Some(json!({"model": "bench", "input": "hi"})),
- )?)
- .await?;
- assert_eq!(responses.status(), StatusCode::OK);
- let responses = json_body(responses).await?;
- assert_eq!(responses["object"], "response");
- assert_eq!(responses["output"][0]["content"][0]["text"], "ok");
- Ok(())
+struct MockUpstream {
+ base_url: String,
+ calls: Arc<Mutex<Vec<Value>>>,
+ task: JoinHandle<()>,
}
-#[tokio::test]
-async fn missing_and_unknown_models_return_client_errors() -> TestResult {
- let app = build_switchyard_router(state_from_yaml(
- r#"
-profiles:
- bench:
- type: noop
-"#,
- )?);
-
- let missing = app
- .clone()
- .oneshot(request(
- "POST",
- "/v1/chat/completions",
- Some(json!({"messages": [{"role": "user", "content": "hi"}]})),
- )?)
- .await?;
- assert_eq!(missing.status(), StatusCode::BAD_REQUEST);
- assert_eq!(
- json_body(missing).await?["error"]["type"],
- "invalid_request_error"
- );
-
- let unknown = app
- .oneshot(request(
- "POST",
- "/v1/chat/completions",
- Some(json!({
- "model": "missing-route",
- "messages": [{"role": "user", "content": "hi"}],
- })),
- )?)
- .await?;
- assert_eq!(unknown.status(), StatusCode::NOT_FOUND);
- assert_eq!(
- json_body(unknown).await?["error"]["type"],
- "model_not_found"
- );
- Ok(())
+impl MockUpstream {
+ async fn start() -> TestResult<Self> {
+ let calls = Arc::new(Mutex::new(Vec::new()));
+ let app = Router::new()
+ .route("/v1/chat/completions", post(upstream_chat))
+ .with_state(Arc::clone(&calls));
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let addr = listener.local_addr()?;
+ let task = tokio::spawn(async move {
+ if let Err(error) = axum::serve(listener, app).await {
+ tracing::error!(error = %error, "mock upstream stopped");
+ }
+ });
+ Ok(Self {
+ base_url: format!("http://{addr}/v1"),
+ calls,
+ task,
+ })
+ }
}
-#[tokio::test]
-async fn malformed_and_non_object_json_return_shared_client_errors() -> TestResult {
- let app = build_switchyard_router(state_from_yaml(
- r#"
-profiles:
- bench:
- type: noop
-"#,
- )?);
-
- for uri in ["/v1/chat/completions", "/v1/messages", "/v1/responses"] {
- let malformed = app.clone().oneshot(raw_request("POST", uri, "{")?).await?;
- assert_eq!(malformed.status(), StatusCode::BAD_REQUEST);
- assert_eq!(json_body(malformed).await?["error"]["code"], "invalid_body");
-
- let non_object = app.clone().oneshot(raw_request("POST", uri, "[]")?).await?;
- assert_eq!(non_object.status(), StatusCode::BAD_REQUEST);
- assert_eq!(
- json_body(non_object).await?["error"]["code"],
- "invalid_body"
- );
+impl Drop for MockUpstream {
+ fn drop(&mut self) {
+ self.task.abort();
}
- Ok(())
}
-#[tokio::test]
-async fn translation_errors_do_not_emit_routing_metadata_headers() -> TestResult {
- let app = build_switchyard_router(state_from_profile("bad", Arc::new(BadTranslationProfile))?);
+async fn upstream_chat(
+ State(calls): State<Arc<Mutex<Vec<Value>>>>,
+ Json(body): Json<Value>,
+) -> HttpResponse {
+ calls.lock().await.push(body.clone());
+ if body["messages"][0]["content"] == "fail" {
+ return (
+ StatusCode::IM_A_TEAPOT,
+ Json(json!({"error": {"message": "upstream rejected request"}})),
+ )
+ .into_response();
+ }
- let response = app
- .oneshot(request(
- "POST",
- "/v1/messages",
- Some(json!({
- "model": "bad",
- "messages": [{"role": "user", "content": "hi"}],
- "max_tokens": 8,
- })),
- )?)
- .await?;
+ let model = body["model"].as_str().unwrap_or("unknown").to_string();
+ if body["stream"].as_bool() == Some(true) {
+ let events = [
+ json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"role": "assistant"}}]}).to_string(),
+ json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "hello"}}]}).to_string(),
+ json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}).to_string(),
+ "[DONE]".to_string(),
+ ];
+ let stream = futures_util::stream::iter(
+ events
+ .into_iter()
+ .map(|data| Ok::<Event, Infallible>(Event::default().data(data))),
+ );
+ return Sse::new(stream).into_response();
+ }
- assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
- assert!(!response
- .headers()
- .keys()
- .any(|name| name.as_str().starts_with("x-model-router-")));
- Ok(())
+ Json(json!({
+ "id": "chatcmpl-test",
+ "object": "chat.completion",
+ "model": model,
+ "choices": [{
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop"
+ }],
+ "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}
+ }))
+ .into_response()
+}
+
+fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<ServerState> {
+ let backend = Backend::OpenAiChat(HttpBackendConfig {
+ base_url: base_url.to_string(),
+ api_key: Some("test-key".to_string()),
+ extra_headers: BTreeMap::new(),
+ });
+ let target_models = routes
+ .iter()
+ .flat_map(|(_, targets)| targets.iter().copied())
+ .collect::<HashSet<_>>();
+ let model_configs = target_models
+ .into_iter()
+ .map(|model| ModelConfig::new(model, backend.clone(), None))
+ .collect::<Vec<_>>();
+ let client: Arc<dyn RoutedLlmClient> = Arc::new(TranslatingLlmClient::new(&model_configs)?);
+ let entries = routes.iter().map(|(route_model, targets)| {
+ let target_set = LlmTargetSet::new(
+ targets
+ .iter()
+ .map(|model| LlmTarget {
+ semantic_name: (*model).to_string(),
+ llm_client: Some(Arc::clone(&client)),
+ })
+ .collect(),
+ );
+ let algorithm: Arc<dyn Algorithm> = Arc::new(Random::new(target_set));
+ ((*route_model).to_string(), algorithm)
+ });
+ Ok(ServerState::new(entries)?)
}
-#[tokio::test]
-async fn target_with_same_id_and_model_is_registered_once() -> TestResult {
- let _stats_guard = stats_guard().await;
- let Some(stub) = HttpStub::start(1)? else {
- log_loopback_bind_skip();
- return Ok(());
- };
- let app = build_switchyard_router(state_from_yaml(&format!(
- r#"
-targets:
- upstream-direct:
- model: upstream-direct
- format: openai
- base_url: {base_url}
-profiles:
- direct-profile:
- type: passthrough
- target: upstream-direct
-"#,
- base_url = stub.base_url
- ))?);
-
- let models = app
- .clone()
- .oneshot(request("GET", "/v1/models", None)?)
- .await?;
- assert_eq!(
- json_body(models).await?["model_pool"],
- json!(["direct-profile", "upstream-direct"])
- );
-
- let response = app
- .oneshot(request(
- "POST",
- "/v1/chat/completions",
- Some(json!({
- "model": "upstream-direct",
- "messages": [{"role": "user", "content": "hi"}],
- })),
- )?)
- .await?;
- assert_eq!(response.status(), StatusCode::OK);
- assert_eq!(stub.requests()?[0]["model"], "upstream-direct");
- Ok(())
+async fn test_app(routes: &[(&str, &[&str])]) -> TestResult<(MockUpstream, Router)> {
+ let upstream = MockUpstream::start().await?;
+ let app = build_switchyard_router(random_state(&upstream.base_url, routes)?);
+ Ok((upstream, app))
}
-#[tokio::test]
-async fn random_routing_profile_reaches_selected_backend_path() -> TestResult {
- let _stats_guard = stats_guard().await;
- reset_stats()?;
- let Some(stub) = HttpStub::start(1)? else {
- log_loopback_bind_skip();
- return Ok(());
+async fn send(app: &Router, method: &str, path: &str, body: Option<Value>) -> TestResult<Response> {
+ let mut builder = HttpRequest::builder().method(method).uri(path);
+ let request_body = if let Some(body) = body {
+ builder = builder.header("content-type", "application/json");
+ Body::from(serde_json::to_vec(&body)?)
+ } else {
+ Body::empty()
};
- let app = build_switchyard_router(state_from_yaml(&format!(
- r#"
-targets:
- strong:
- model: upstream-strong
- format: openai
- base_url: {base_url}
- weak:
- model: upstream-weak
- format: openai
- base_url: {base_url}
-profiles:
- random:
- type: random-routing
- strong: strong
- weak: weak
- strong_probability: 0.0000004
- rng_seed: 7
-"#,
- base_url = stub.base_url
- ))?);
-
- let response = app
- .oneshot(request(
- "POST",
- "/v1/chat/completions",
- Some(json!({
- "model": "random",
- "messages": [{"role": "user", "content": "hi"}],
- })),
- )?)
- .await?;
- assert_eq!(response.status(), StatusCode::OK);
- for (name, expected) in [
- ("x-model-router-selected-model", "upstream-weak"),
- ("x-model-router-selected-tier", "weak"),
- ("x-model-router-version", "random-routing:v1"),
- ("x-model-router-tolerance", "0.0000004"),
- ] {
- assert_eq!(header(&response, name), Some(expected));
- }
- assert!(header(&response, "x-model-router-rationale")
- .is_some_and(|value| value.contains("strong_probability 0.0000004; selected weak")));
+ let response = app.clone().oneshot(builder.body(request_body)?).await?;
+ let status = response.status();
+ let headers = response.headers().clone();
+ let bytes = response.into_body().collect().await?.to_bytes();
+ Ok(Response {
+ status,
+ headers,
+ bytes,
+ })
+}
- let seen = stub.requests()?;
- assert_eq!(seen.len(), 1);
- assert_eq!(seen[0]["model"], "upstream-weak");
- Ok(())
+struct Response {
+ status: StatusCode,
+ headers: axum::http::HeaderMap,
+ bytes: Bytes,
}
-#[tokio::test]
-async fn latency_service_profile_reaches_configured_backend_path() -> TestResult {
- let _stats_guard = stats_guard().await;
- reset_stats()?;
- let Some(stub) = HttpStub::start(1)? else {
- log_loopback_bind_skip();
- return Ok(());
- };
- let app = build_switchyard_router(state_from_yaml(&format!(
- r#"
-targets:
- fast:
- model: upstream-fast
- format: openai
- base_url: {base_url}
-profiles:
- latency:
- type: latency-service
- latency_service_url: http://latency.local
- targets: [fast]
-"#,
- base_url = stub.base_url
- ))?);
-
- let response = app
- .oneshot(request(
- "POST",
- "/v1/chat/completions",
- Some(json!({
- "model": "latency",
- "messages": [{"role": "user", "content": "hi"}],
- })),
- )?)
- .await?;
- assert_eq!(response.status(), StatusCode::OK);
+impl Response {
+ fn json(&self) -> TestResult<Value> {
+ Ok(serde_json::from_slice(&self.bytes)?)
+ }
- let seen = stub.requests()?;
- assert_eq!(seen.len(), 1);
- assert_eq!(seen[0]["model"], "upstream-fast");
- Ok(())
+ fn text(&self) -> TestResult<&str> {
+ Ok(std::str::from_utf8(&self.bytes)?)
+ }
}
#[tokio::test]
-async fn stats_endpoints_use_components_v2_global_accumulator() -> TestResult {
- let _stats_guard = stats_guard().await;
- reset_stats()?;
- profile_stats_accumulator().record_success("served-model", Some(12.0), Some("strong"))?;
- let app = build_switchyard_router(state_from_yaml(
- r#"
-profiles:
- bench:
- type: noop
-"#,
- )?);
-
- let stats = app
- .clone()
- .oneshot(request("GET", "/v1/routing/stats", None)?)
- .await?;
- assert_eq!(stats.status(), StatusCode::OK);
+async fn routes_dispatch_and_discovery_endpoints_are_stable() -> TestResult {
+ let (upstream, app) = test_app(&[
+ ("switchyard/coding", &["model/code"]),
+ ("switchyard/general", &["model/general"]),
+ ])
+ .await?;
+
+ let health = send(&app, "GET", "/health", None).await?;
+ assert_eq!(health.status, StatusCode::OK);
+ assert_eq!(health.json()?, json!({"status": "ok"}));
+
+ let models = send(&app, "GET", "/v1/models", None).await?;
+ assert_eq!(models.status, StatusCode::OK);
assert_eq!(
- json_body(stats).await?["models"]["served-model"]["calls"],
- 1
+ models.json()?["model_pool"],
+ json!(["switchyard/coding", "switchyard/general"])
);
- let reset = app
- .clone()
- .oneshot(request("POST", "/v1/routing/stats/reset", None)?)
- .await?;
- assert_eq!(reset.status(), StatusCode::OK);
- assert_eq!(json_body(reset).await?, json!({"status": "reset"}));
-
- let after = app.oneshot(request("GET", "/v1/stats", None)?).await?;
- assert_eq!(json_body(after).await?["models"], json!({}));
- Ok(())
-}
+ let missing = send(&app, "GET", "/missing", None).await?;
+ assert_eq!(missing.status, StatusCode::NOT_FOUND);
+ assert_eq!(missing.json()?["error"]["code"], "endpoint_not_found");
-#[tokio::test]
-async fn openai_streams_are_sse_framed_with_done() -> TestResult {
- let app = build_switchyard_router(state_from_profile(
- "stream",
- Arc::new(StreamProfile {
- kind: StreamKind::OpenAi,
- routing_metadata: None,
- }),
- )?);
-
- let response = app
- .oneshot(request(
+ for (route_model, target_model) in [
+ ("switchyard/general", "model/general"),
+ ("switchyard/coding", "model/code"),
+ ] {
+ let response = send(
+ &app,
"POST",
"/v1/chat/completions",
Some(json!({
- "model": "stream",
- "messages": [{"role": "user", "content": "hi"}],
- "stream": true,
+ "model": route_model,
+ "messages": [{"role": "user", "content": "hi"}]
})),
- )?)
+ )
.await?;
- assert_eq!(response.status(), StatusCode::OK);
- let body = text_body(response).await?;
- assert!(body.contains("data: {\"choices\":[{\"delta\":{\"content\":\"hello\"}"));
- assert!(body.contains("data: [DONE]"));
+ assert_eq!(response.status, StatusCode::OK);
+ assert_eq!(
+ response
+ .headers
+ .get("x-model-router-selected-model")
+ .and_then(|value| value.to_str().ok()),
+ Some(target_model)
+ );
+ }
+
+ let calls = upstream.calls.lock().await;
+ assert_eq!(calls[0]["model"], "model/general");
+ assert_eq!(calls[1]["model"], "model/code");
Ok(())
}
#[tokio::test]
-async fn openai_streams_include_routing_metadata_headers() -> TestResult {
- let app = build_switchyard_router(state_from_profile(
- "stream",
- Arc::new(StreamProfile {
- kind: StreamKind::OpenAi,
- routing_metadata: Some(RoutingMetadata {
- selected_model: Some("served-model".to_string()),
- selected_tier: Some("weak".to_string()),
- confidence: Some(0.0000004),
- router_version: Some("test-router:v1".to_string()),
- tolerance: Some(0.0000004),
- rationale: Some("line\nbreak".to_string()),
- }),
- }),
- )?);
+async fn all_inbound_formats_run_libsy_and_return_the_caller_format() -> TestResult {
+ let (upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
- let response = app
- .oneshot(request(
- "POST",
+ let cases = [
+ (
"/v1/chat/completions",
- Some(json!({
- "model": "stream",
- "messages": [{"role": "user", "content": "hi"}],
- "stream": true,
- })),
- )?)
- .await?;
- assert_eq!(response.status(), StatusCode::OK);
- for (name, expected) in [
- ("x-model-router-selected-model", "served-model"),
- ("x-model-router-confidence", "0.0000004"),
- ("x-model-router-rationale", "line break"),
- ] {
- assert_eq!(header(&response, name), Some(expected));
- }
- let body = text_body(response).await?;
- assert!(body.contains("data: [DONE]"));
- Ok(())
-}
-
-#[tokio::test]
-async fn anthropic_streams_are_named_sse_without_done() -> TestResult {
- let app = build_switchyard_router(state_from_profile(
- "stream",
- Arc::new(StreamProfile {
- kind: StreamKind::Anthropic,
- routing_metadata: None,
- }),
- )?);
-
- let response = app
- .oneshot(request(
- "POST",
+ json!({
+ "model": ROUTE_MODEL,
+ "messages": [{"role": "user", "content": "hi"}]
+ }),
+ ),
+ (
"/v1/messages",
- Some(json!({
- "model": "stream",
+ json!({
+ "model": ROUTE_MODEL,
"max_tokens": 16,
- "messages": [{"role": "user", "content": "hi"}],
- "stream": true,
- })),
- )?)
- .await?;
- assert_eq!(response.status(), StatusCode::OK);
- let body = text_body(response).await?;
- assert!(body.contains("event: message_start"));
- assert!(body.contains("\"type\":\"message_start\""));
- assert!(!body.contains("[DONE]"));
- Ok(())
-}
+ "messages": [{"role": "user", "content": "hi"}]
+ }),
+ ),
+ (
+ "/v1/responses",
+ json!({"model": ROUTE_MODEL, "input": "hi"}),
+ ),
+ ];
-#[tokio::test]
-async fn endpoint_metadata_is_passed_to_profiles() -> TestResult {
- let captured = Arc::new(Mutex::new(None));
- let app = build_switchyard_router(state_from_profile(
- "capture",
- Arc::new(CaptureProfile {
- captured: Arc::clone(&captured),
- }),
- )?);
-
- let response = app
- .oneshot(
- Request::builder()
- .method("POST")
- .uri("/v1/messages")
- .header("content-type", "application/json")
- .header("X-Request-ID", "req-123")
- .header("X-Switchyard-Trace", "trace-a")
- .body(Body::from(
- json!({
- "model": "capture",
- "max_tokens": 16,
- "messages": [{"role": "user", "content": "hi"}],
- })
- .to_string(),
- ))?,
- )
- .await?;
- assert_eq!(response.status(), StatusCode::OK);
-
- let input = captured
- .lock()
- .map_err(|_| "captured input mutex poisoned")?
- .clone()
- .ok_or("profile should have received input")?;
- assert_eq!(input.request.model(), Some("capture"));
- assert_eq!(
- input.metadata.request_id.as_ref().map(|id| id.as_str()),
- Some("req-123")
- );
+ let mut responses = Vec::new();
+ for (path, body) in cases {
+ responses.push(send(&app, "POST", path, Some(body)).await?);
+ }
+
+ assert!(responses
+ .iter()
+ .all(|response| response.status == StatusCode::OK));
assert_eq!(
- input.metadata.inbound_format,
- Some(ChatRequestType::Anthropic)
+ responses[0].json()?["choices"][0]["message"]["content"],
+ "ok"
);
+ assert_eq!(responses[1].json()?["content"][0]["text"], "ok");
assert_eq!(
- input
- .metadata
- .headers
- .get("x-switchyard-trace")
- .map(Vec::as_slice),
- Some(&["trace-a".to_string()][..])
+ responses[2].json()?["output"][0]["content"][0]["text"],
+ "ok"
);
- Ok(())
-}
-
-/// Test profile that emits one deterministic stream event for SSE framing checks.
-#[derive(Clone)]
-struct StreamProfile {
- kind: StreamKind,
- routing_metadata: Option<RoutingMetadata>,
-}
-
-/// Stream format variant emitted by `StreamProfile`.
-#[derive(Clone, Copy)]
-enum StreamKind {
- OpenAi,
- Anthropic,
-}
-
-#[async_trait]
-impl Profile for StreamProfile {
- async fn run(&self, _input: ProfileInput) -> Result<ProfileResponse> {
- let response = match self.kind {
- StreamKind::OpenAi => ChatResponse::OpenAiStream(Box::pin(futures_util::stream::iter(
- [Ok(StreamEvent::Json(json!({
- "id": "chatcmpl-test",
- "object": "chat.completion.chunk",
- "model": "served-model",
- "choices": [{
- "index": 0,
- "delta": {"content": "hello"},
- "finish_reason": null,
- }],
- })))],
- ))),
- StreamKind::Anthropic => ChatResponse::AnthropicStream(Box::pin(
- futures_util::stream::iter([Ok(StreamEvent::Json(json!({
- "type": "message_start",
- "message": {
- "id": "msg-test",
- "type": "message",
- "role": "assistant",
- "content": [],
- "model": "claude-test",
- "stop_reason": null,
- "stop_sequence": null,
- "usage": {"input_tokens": 1, "output_tokens": 0},
- },
- })))]),
- )),
- };
- Ok(match &self.routing_metadata {
- Some(metadata) => ProfileResponse::with_routing_metadata(response, metadata.clone()),
- None => ProfileResponse::from(response),
- })
- }
-}
-
-struct CaptureProfile {
- captured: Arc<Mutex<Option<ProfileInput>>>,
-}
-
-struct BadTranslationProfile;
-
-#[async_trait]
-impl Profile for BadTranslationProfile {
- async fn run(&self, _input: ProfileInput) -> Result<ProfileResponse> {
- Ok(ProfileResponse::with_routing_metadata(
- ChatResponse::openai_completion(json!("not an OpenAI response object")),
- RoutingMetadata {
- selected_model: Some("bad-upstream".to_string()),
- selected_tier: Some("weak".to_string()),
- router_version: Some("test-router:v1".to_string()),
- ..RoutingMetadata::default()
- },
- ))
- }
-}
-
-#[async_trait]
-impl Profile for CaptureProfile {
- async fn run(&self, input: ProfileInput) -> Result<ProfileResponse> {
- {
- let mut captured = self
- .captured
- .lock()
- .map_err(|_| SwitchyardError::Other("captured input mutex poisoned".to_string()))?;
- *captured = Some(input);
- }
- Ok(ChatResponse::anthropic_completion(json!({
- "id": "msg-capture",
- "type": "message",
- "role": "assistant",
- "content": [{"type": "text", "text": "captured"}],
- "model": "capture",
- "stop_reason": "end_turn",
- "stop_sequence": null,
- "usage": {"input_tokens": 1, "output_tokens": 1},
- }))
- .into())
- }
-}
-
-/// In-process HTTP stub that records JSON request bodies from backend calls.
-struct HttpStub {
- base_url: String,
- addr: SocketAddr,
- expected_requests: usize,
- requests: Arc<Mutex<Vec<Value>>>,
- handle: Option<thread::JoinHandle<()>>,
-}
-
-impl HttpStub {
- /// Binds an ephemeral port and accepts the expected number of stub requests.
- fn start(expected_requests: usize) -> TestResult<Option<Self>> {
- let listener = match StdTcpListener::bind("127.0.0.1:0") {
- Ok(listener) => listener,
- Err(error) if error.kind() == ErrorKind::PermissionDenied => return Ok(None),
- Err(error) => return Err(error.into()),
- };
- let addr = listener.local_addr()?;
- let requests = Arc::new(Mutex::new(Vec::new()));
- let thread_requests = Arc::clone(&requests);
- let handle = thread::spawn(move || {
- for _ in 0..expected_requests {
- let Ok((mut stream, _addr)) = listener.accept() else {
- return;
- };
- if let Ok(body) = read_http_body(&mut stream) {
- if let Ok(value) = serde_json::from_slice::<Value>(&body) {
- if let Ok(mut requests) = thread_requests.lock() {
- requests.push(value);
- }
- }
- }
- let response = json!({
- "id": "chatcmpl-stub",
- "object": "chat.completion",
- "model": "stub",
- "choices": [{
- "index": 0,
- "message": {"role": "assistant", "content": "stub-ok"},
- "finish_reason": "stop",
- }],
- "usage": {
- "prompt_tokens": 1,
- "completion_tokens": 1,
- "total_tokens": 2,
- },
- })
- .to_string();
- let _ = write!(
- stream,
- "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
- response.len(),
- response
- );
- }
- });
-
- Ok(Some(Self {
- base_url: format!("http://{addr}/v1"),
- addr,
- expected_requests,
- requests,
- handle: Some(handle),
- }))
- }
-
- /// Returns the JSON request bodies captured by the stub thread.
- fn requests(&self) -> TestResult<Vec<Value>> {
- self.requests
- .lock()
- .map(|requests| requests.clone())
- .map_err(|_| "stub request mutex poisoned".into())
- }
-}
-
-impl Drop for HttpStub {
- fn drop(&mut self) {
- // Wake pending accepts before joining the stub thread.
- for _ in 0..self.expected_requests {
- let _ = TcpStream::connect(self.addr);
- }
- if let Some(handle) = self.handle.take() {
- let _ = handle.join();
- }
- }
-}
-
-fn read_http_body(stream: &mut std::net::TcpStream) -> TestResult<Vec<u8>> {
- let mut buffer = Vec::new();
- let mut header_end = None;
- while header_end.is_none() {
- let mut chunk = [0; 1024];
- let read = stream.read(&mut chunk)?;
- if read == 0 {
- break;
- }
- buffer.extend_from_slice(&chunk[..read]);
- header_end = find_bytes(&buffer, b"\r\n\r\n").map(|index| index + 4);
- }
-
- let Some(body_start) = header_end else {
- return Err("HTTP request headers were incomplete".into());
- };
- let headers = std::str::from_utf8(&buffer[..body_start])?;
- let content_length = headers
- .lines()
- .find_map(|line| {
- let (name, value) = line.split_once(':')?;
- name.eq_ignore_ascii_case("content-length")
- .then(|| value.trim().parse::<usize>().ok())
- .flatten()
- })
- .unwrap_or(0);
-
- while buffer.len() < body_start + content_length {
- let mut chunk = [0; 1024];
- let read = stream.read(&mut chunk)?;
- if read == 0 {
- break;
- }
- buffer.extend_from_slice(&chunk[..read]);
+ for response in &responses {
+ assert_eq!(
+ response
+ .headers
+ .get("x-model-router-selected-model")
+ .and_then(|value| value.to_str().ok()),
+ Some("model/a")
+ );
}
- Ok(buffer[body_start..body_start + content_length].to_vec())
-}
-
-fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option<usize> {
- haystack
- .windows(needle.len())
- .position(|window| window == needle)
-}
-
-fn state_from_yaml(input: &str) -> TestResult<ServerState> {
- let plan = parse_profile_config_str(input, ProfileConfigFormat::Yaml)?.resolve()?;
- Ok(ServerState::from_plan(&plan)?)
-}
-
-fn state_from_profile(model: &'static str, profile: Arc<dyn Profile>) -> TestResult<ServerState> {
- let registry = ProfileRegistry::from_profiles([(
- ModelId::from_static(model),
- profile,
- model.to_string(),
- )])?;
- Ok(ServerState::new(registry))
-}
-
-fn request(method: &str, uri: &str, body: Option<Value>) -> TestResult<Request<Body>> {
- let builder = Request::builder()
- .method(method)
- .uri(uri)
- .header("content-type", "application/json");
- let body = body.map_or_else(Body::empty, |body| Body::from(body.to_string()));
- Ok(builder.body(body)?)
-}
-
-fn raw_request(method: &str, uri: &str, body: &str) -> TestResult<Request<Body>> {
- let builder = Request::builder()
- .method(method)
- .uri(uri)
- .header("content-type", "application/json");
- Ok(builder.body(Body::from(body.to_string()))?)
-}
-
-async fn json_body(response: axum::response::Response) -> TestResult<Value> {
- let bytes = response.into_body().collect().await?.to_bytes();
- Ok(serde_json::from_slice(&bytes)?)
-}
-
-async fn text_body(response: axum::response::Response) -> TestResult<String> {
- let bytes = response.into_body().collect().await?.to_bytes();
- Ok(String::from_utf8(bytes.to_vec())?)
-}
-
-fn header<'a>(response: &'a axum::response::Response, name: &str) -> Option<&'a str> {
- response
- .headers()
- .get(name)
- .and_then(|value| value.to_str().ok())
+ let calls = upstream.calls.lock().await;
+ assert_eq!(calls.len(), 3);
+ assert!(calls.iter().all(|call| call["model"] == "model/a"));
+ Ok(())
}
-fn reset_stats() -> Result<()> {
- profile_stats_accumulator().reset()
+#[tokio::test]
+async fn streaming_response_is_framed_for_the_inbound_api() -> TestResult {
+ let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
+
+ let response = send(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({
+ "model": ROUTE_MODEL,
+ "messages": [{"role": "user", "content": "hi"}],
+ "stream": true
+ })),
+ )
+ .await?;
+
+ assert_eq!(response.status, StatusCode::OK);
+ assert!(response.text()?.contains("hello"));
+ assert!(response.text()?.contains("data: [DONE]"));
+ Ok(())
}
-/// Serializes tests that touch the global profile stats accumulator.
-///
-/// The lock is initialized once for the test process and held by each caller
-/// until the returned guard is dropped.
-async fn stats_guard() -> tokio::sync::MutexGuard<'static, ()> {
- static LOCK: OnceLock<tokio::sync::Mutex<()>> = OnceLock::new();
- LOCK.get_or_init(|| tokio::sync::Mutex::new(()))
- .lock()
- .await
-}
+#[tokio::test]
+async fn request_and_upstream_errors_use_the_canonical_envelope() -> TestResult {
+ let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
+
+ let unknown = send(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({
+ "model": "other",
+ "messages": [{"role": "user", "content": "hi"}]
+ })),
+ )
+ .await?;
+ assert_eq!(unknown.status, StatusCode::NOT_FOUND);
+ assert_eq!(unknown.json()?["error"]["code"], "model_not_found");
+
+ let missing_model = send(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({"messages": [{"role": "user", "content": "hi"}]})),
+ )
+ .await?;
+ assert_eq!(missing_model.status, StatusCode::BAD_REQUEST);
+ assert_eq!(
+ missing_model.json()?["error"]["code"],
+ "invalid_request_error"
+ );
-fn log_loopback_bind_skip() {
- eprintln!("SKIP: permission denied binding loopback socket");
+ let upstream_error = send(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({
+ "model": ROUTE_MODEL,
+ "messages": [{"role": "user", "content": "fail"}]
+ })),
+ )
+ .await?;
+ assert_eq!(upstream_error.status, StatusCode::IM_A_TEAPOT);
+ assert_eq!(upstream_error.json()?["error"]["code"], "upstream_error");
+ Ok(())
}
-
-type TestResult<T = ()> = std::result::Result<T, Box<dyn std::error::Error + Send + Sync>>;
diff --git a/docs/cli_reference.md b/docs/cli_reference.md
index c9b47cda1..4c3b98f4a 100644
--- a/docs/cli_reference.md
+++ b/docs/cli_reference.md
@@ -21,7 +21,7 @@ These apply to the top-level `switchyard` command, before any verb.
|---|---|
| `--version` | Print the installed Switchyard version (`switchyard X.Y.Z`) and exit. Reads the version from the installed package metadata. |
| `--routing-profiles PATH` / `-c PATH` | Deprecated legacy [Routing](#routing) bundle applied to `serve`, `launch`, and `configure`. Pass before the verb; separate with `--` for clarity. |
-| `--enable-rl-logging` | Write local [RL trace logs](#rl-trace-logging) (one `message_history` JSON file per turn) for `launch` and `serve` route-bundle sessions. Pass before the verb: `switchyard --enable-rl-logging launch claude`. Rejected by `serve --config` (the Rust profile server has no Python processor chain). |
+| `--enable-rl-logging` | Write local [RL trace logs](#rl-trace-logging) (one `message_history` JSON file per turn) for `launch` and `serve` route-bundle sessions. Pass before the verb: `switchyard --enable-rl-logging launch claude`. Rejected by `serve --config`. |
| `--rl-log-dir DIR` | Output directory for `--enable-rl-logging` traces (default: `./rl_data`). No effect without `--enable-rl-logging`. |
## Cross-cutting flag families
@@ -147,8 +147,7 @@ matches the pre-1.0 trace format:
```
Turns without an assistant choice (e.g. upstream errors) are skipped. The flag
-works on `launch` and on `serve` with a route bundle; `serve --config` (the Rust
-profile server) rejects it, since it has no Python processor chain to attach to.
+works on `launch` and on `serve` with a route bundle; `serve --config` rejects it.
### Transport (server verbs)
@@ -205,7 +204,7 @@ because the built-in default is `https://openrouter.ai/api/v1`.
## `switchyard serve`
-Serve a long-running proxy from a Switchyard v2 **profile config**: one YAML/JSON/TOML file declaring `endpoints`, `targets`, and `profiles`. Files whose profiles are all Rust-defined use the Rust profile server. Files that include Python-defined profiles use the Python FastAPI adapter, while keeping the same endpoint paths. Each profile id and each target id is exposed as a model on `GET /v1/models`, so a client selects a profile by setting the request `model` to that id.
+Serve a long-running proxy from a Switchyard v2 **profile config**: one YAML/JSON/TOML file declaring `endpoints`, `targets`, and `profiles`. Rust-defined and Python-defined profiles run through the same Python HTTP application and expose the same endpoint paths. Each profile id and each target id is exposed as a model on `GET /v1/models`, so a client selects a profile by setting the request `model` to that id.
The server exposes the OpenAI Chat Completions (`/v1/chat/completions`), Anthropic Messages (`/v1/messages`), and OpenAI Responses (`/v1/responses`) APIs on the same host and port.
diff --git a/docs/internal/metrics_reference.md b/docs/internal/metrics_reference.md
index 859c3a9fd..3dcdc18f4 100644
--- a/docs/internal/metrics_reference.md
+++ b/docs/internal/metrics_reference.md
@@ -16,7 +16,7 @@ a drop-in scrape config and starter alert rules.
!!! note "Availability: legacy route-bundle serve only"
`GET /metrics` is served by the **legacy route-bundle path**
- (`switchyard --routing-profiles PATH serve`). The recommended v2 profile server
+ (`switchyard --routing-profiles PATH serve`). The v2 profile-config path
(`switchyard serve --config`) returns **404** for `/metrics` — use `GET /v1/stats`
(or its alias `GET /v1/routing/stats`) on that path instead.
diff --git a/switchyard/cli/switchyard_cli.py b/switchyard/cli/switchyard_cli.py
index cac0ff3d3..a7fb6ebc0 100644
--- a/switchyard/cli/switchyard_cli.py
+++ b/switchyard/cli/switchyard_cli.py
@@ -99,7 +99,6 @@ from switchyard.server.server_util import (
add_transport_args,
build_and_serve,
load_secrets,
- resolve_port,
resolve_rl_log_dir,
)
@@ -144,17 +143,6 @@ def _warn_deprecated_saved_route_bundle() -> None:
)
-def _warn_deprecated_python_profile_server(python_profiles: Sequence[str]) -> None:
- _print_deprecation_warning(
- "Python-defined profile serving",
- details=(
- "This config is using the Python FastAPI adapter.",
- f"Python profile(s): {', '.join(python_profiles)}.",
- "Prefer Rust-defined components-v2 profiles for new serve configs.",
- ),
- )
-
-
class _IntakeEnabledAction(argparse.Action):
"""Store the normalized Intake enable flag and warn on the deprecated alias."""
@@ -468,7 +456,7 @@ def _cmd_serve(args: argparse.Namespace) -> None:
def _cmd_serve_profile_config(args: argparse.Namespace) -> None:
- """Serve a components-v2 profile config."""
+ """Serve a v2 profile config through the Python HTTP application."""
if args.routing_profiles:
raise SystemExit(
"serve --config cannot be combined with --routing-profiles; "
@@ -494,47 +482,14 @@ def _cmd_serve_profile_config(args: argparse.Namespace) -> None:
raise SystemExit("serve --config does not support Intake options yet.")
if getattr(args, "enable_rl_logging", False):
raise SystemExit(
- "serve --config does not support --enable-rl-logging: the Rust "
- "profile server has no Python processor chain to attach the trace "
- "logger to. Use serve --routing-profiles for local RL trace logging."
+ "serve --config does not support --enable-rl-logging. "
+ "Use serve --routing-profiles for local RL trace logging."
)
- # Inspect first so files containing only Rust-defined profiles can use the
- # Rust server path, while files with Python-defined profiles use FastAPI.
- from switchyard.lib.profiles.loader import python_profile_ids
-
- try:
- python_profiles = python_profile_ids(args.config)
- except Exception as exc:
- raise SystemExit(
- "serve --config: failed to inspect profile config for "
- f"Python-defined profiles: {exc}"
- ) from exc
- if python_profiles:
- _cmd_serve_mixed_profile_config(args, python_profiles)
- return
-
- from switchyard_rust.server import run_profile_server
-
- port = args.port if isinstance(args.port, int) else resolve_port()
- logger.info(
- "Switchyard components-v2 profile config loaded from %s",
- args.config,
- )
- run_profile_server(args.config, args.host, port)
-
-
-def _cmd_serve_mixed_profile_config(
- args: argparse.Namespace,
- python_profiles: list[str],
-) -> None:
- """Serve a config containing Python-defined profiles through FastAPI."""
- _warn_deprecated_python_profile_server(python_profiles)
table = _profile_config_route_table(args.config)
logger.info(
- "Switchyard profile config loaded from %s with Python-defined profile(s): %s",
+ "Switchyard profile config loaded from %s",
args.config,
- ", ".join(python_profiles),
)
build_and_serve(args, table, inbound_default="both")
@@ -928,10 +883,7 @@ def _build_parser() -> argparse.ArgumentParser:
default=None,
metavar="PATH",
help=(
- "Path to a Switchyard v2 profile config (YAML, JSON, or TOML). "
- "Files containing only Rust-defined profiles use the Rust profile "
- "server; files with Python-defined profiles use the Python FastAPI "
- "adapter."
+ "Path to a Switchyard v2 profile config (YAML, JSON, or TOML)."
),
)
serve.add_argument(
diff --git a/switchyard/lib/processors/random_routing_request_processor.py b/switchyard/lib/processors/random_routing_request_processor.py
index 48e2e8abf..e9320ef84 100644
--- a/switchyard/lib/processors/random_routing_request_processor.py
+++ b/switchyard/lib/processors/random_routing_request_processor.py
@@ -15,8 +15,6 @@ class RandomRoutingRequestProcessor:
"""Rewrite requests to the randomly selected strong or weak target.
This is a compatibility component for the current Python profile chain.
- New Rust serving code uses components-v2 profiles directly instead of
- exporting a Rust request-processor object.
"""
def __init__(self, config: RandomRoutingProcessorConfig) -> None:
diff --git a/switchyard/lib/profiles/loader.py b/switchyard/lib/profiles/loader.py
index 713615be1..ee56925be 100644
--- a/switchyard/lib/profiles/loader.py
+++ b/switchyard/lib/profiles/loader.py
@@ -29,7 +29,7 @@ from switchyard_rust.profiles import (
parse_profile_config_path,
)
-__all__ = ["load_profiles", "load_profiles_and_targets", "python_profile_ids"]
+__all__ = ["load_profiles", "load_profiles_and_targets"]
_PROFILE_TARGET = "profile_target"
@@ -66,17 +66,6 @@ def load_profiles_and_targets(
return built, _targets(plan)
-def python_profile_ids(path: str | Path) -> list[str]:
- """Return the ids of Python-defined profiles declared in ``path``.
-
- Used by ``serve --config`` to select the Rust server for files containing
- only Rust-defined profiles and the Python FastAPI adapter for files that
- include Python-defined profiles.
- """
- _register_shipped_python_profiles()
- return sorted(_python_profiles(parse_profile_config_path(path)))
-
-
class _RustProfileRunner:
"""Adapts an erased Rust ``Profile`` to the ``run(input)`` runner contract."""
diff --git a/switchyard_rust/core.py b/switchyard_rust/core.py
index 3633cfc2c..a7859405e 100644
--- a/switchyard_rust/core.py
+++ b/switchyard_rust/core.py
@@ -252,15 +252,6 @@ class _NativeModule(Protocol):
SessionCache: type[Any]
session_key_from_body: Any
- def run_profile_server(
- self,
- config_path: str,
- host: str,
- port: int,
- backlog: int,
- dry_run: bool,
- ) -> None: ...
-
def _ensure_switchyard_version_env() -> None:
if os.environ.get("SWITCHYARD_VERSION", "").strip():
diff --git a/switchyard_rust/server.py b/switchyard_rust/server.py
deleted file mode 100644
index 7ad6c82c1..000000000
--- a/switchyard_rust/server.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Rust components-v2 profile server bindings."""
-
-from switchyard_rust.core import _load_native
-
-
-def run_profile_server(
- config_path: str,
- host: str = "127.0.0.1",
- port: int = 4000,
- backlog: int = 65_535,
- dry_run: bool = False,
-) -> None:
- """Run the Rust components-v2 profile server from an installed package."""
- _load_native().run_profile_server(config_path, host, port, backlog, dry_run)
-
-
-__all__ = ["run_profile_server"]
diff --git a/tests/test_profile_migration.py b/tests/test_profile_migration.py
index e0a14d94f..58446567c 100644
--- a/tests/test_profile_migration.py
+++ b/tests/test_profile_migration.py
@@ -3,13 +3,11 @@
"""Migration tests for replacing legacy serving paths with components-v2 profiles."""
-import argparse
import errno
import json
import threading
from collections.abc import Iterator
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-from pathlib import Path
from typing import Any
import pytest
@@ -535,73 +533,3 @@ async def test_profile_runner_uses_same_profile_config(
assert response.body["model"] == "provider/direct"
assert response.body["mock_path"] == "/v2-direct/v1/chat/completions"
assert [call["body"]["model"] for call in openai_stub.requests] == ["provider/direct"]
-
-
-def test_cli_serve_config_delegates_to_rust_profile_server(
- mocker: MockerFixture,
- tmp_path: Path,
-) -> None:
- import switchyard.cli.switchyard_cli as cli
- import switchyard_rust.server as rust_server
-
- config_path = tmp_path / "profiles.yaml"
- config_path.write_text("profiles:\n bench:\n type: noop\n", encoding="utf-8")
- captured: dict[str, object] = {}
-
- def _fake_run_profile_server(
- config_path: str,
- host: str = "127.0.0.1",
- port: int = 4000,
- backlog: int = 65_535,
- dry_run: bool = False,
- ) -> None:
- captured["config_path"] = config_path
- captured["host"] = host
- captured["port"] = port
- captured["backlog"] = backlog
- captured["dry_run"] = dry_run
-
- mocker.patch.object(
- rust_server,
- "run_profile_server",
- side_effect=_fake_run_profile_server,
- )
-
- def _fail_build_and_serve(
- args: argparse.Namespace,
- switchyard: object,
- inbound_default: str = "openai",
- disable_backend_streaming: bool = False,
- extra_endpoints: list[object] | None = None,
- ) -> None:
- _ = (
- args,
- switchyard,
- inbound_default,
- disable_backend_streaming,
- extra_endpoints,
- )
- pytest.fail("route-bundle server should not run")
-
- mocker.patch.object(cli, "build_and_serve", side_effect=_fail_build_and_serve)
-
- args = cli._build_parser().parse_args(
- [
- "serve",
- "--config",
- str(config_path),
- "--host",
- "127.0.0.1",
- "--port",
- "4555",
- ]
- )
- args.func(args)
-
- assert captured == {
- "config_path": str(config_path),
- "host": "127.0.0.1",
- "port": 4555,
- "backlog": 65_535,
- "dry_run": False,
- }
diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.py
index 86860c729..cafe39e10 100644
--- a/tests/test_route_bundle.py
+++ b/tests/test_route_bundle.py
@@ -421,73 +421,6 @@ def test_serve_subcommand_hands_table_to_server(
assert captured["inbound_default"] == "both"
-def test_serve_config_delegates_to_rust_profile_server(
- mocker: Any,
- tmp_path,
-) -> None:
- import switchyard.cli.switchyard_cli as cli
- import switchyard_rust.server as rust_server
-
- config_path = tmp_path / "profiles.yaml"
- config_path.write_text("profiles:\n bench:\n type: noop\n")
- captured: dict[str, Any] = {}
-
- def _fake_run_profile_server(
- config_path: str,
- host: str = "127.0.0.1",
- port: int = 4000,
- backlog: int = 65_535,
- dry_run: bool = False,
- ) -> None:
- captured["config_path"] = config_path
- captured["host"] = host
- captured["port"] = port
- captured["backlog"] = backlog
- captured["dry_run"] = dry_run
-
- mocker.patch.object(
- rust_server,
- "run_profile_server",
- side_effect=_fake_run_profile_server,
- )
-
- def _fail_build_and_serve(
- args: argparse.Namespace,
- switchyard: object,
- *,
- inbound_default: str = "openai",
- disable_backend_streaming: bool = False,
- **_kwargs: object,
- ) -> None:
- _ = (args, switchyard, inbound_default, disable_backend_streaming)
- pytest.fail("route-bundle server should not run")
-
- mocker.patch.object(
- cli,
- "build_and_serve",
- side_effect=_fail_build_and_serve,
- )
-
- args = cli._build_parser().parse_args([
- "serve",
- "--config",
- str(config_path),
- "--host",
- "127.0.0.1",
- "--port",
- "4555",
- ])
- args.func(args)
-
- assert captured == {
- "config_path": str(config_path),
- "host": "127.0.0.1",
- "port": 4555,
- "backlog": 65_535,
- "dry_run": False,
- }
-
-
def test_serve_config_and_routing_profiles_are_mutually_exclusive(
tmp_path,
) -> None:
diff --git a/tests/test_serve_profile_config.py b/tests/test_serve_profile_config.py
index 29809fe34..299cb4efa 100644
--- a/tests/test_serve_profile_config.py
+++ b/tests/test_serve_profile_config.py
@@ -1,13 +1,7 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-"""Tests for serving a v2 profile config via `serve --config`.
-
-`dry_run` exercises the full load -> resolve -> build -> registry path of the
-Rust profile server without binding a socket. Files with Python-defined profiles
-use the Python FastAPI adapter so both profile implementations are routable on
-the same paths.
-"""
+"""Tests for serving a v2 profile config via `serve --config`."""
import argparse
import json
@@ -20,36 +14,7 @@ import pytest
from fastapi.testclient import TestClient
from switchyard.cli.switchyard_cli import _cmd_serve_profile_config, _profile_config_route_table
-from switchyard.lib.profiles.loader import python_profile_ids
from switchyard.server.switchyard_app import build_switchyard_app
-from switchyard_rust.core import SwitchyardConfigError
-from switchyard_rust.server import run_profile_server
-
-_RUST_CONFIG = """
-targets:
- strong:
- model: provider/strong
- format: openai
- base_url: http://127.0.0.1:9/strong/v1
- api_key: test-key
- weak:
- model: provider/weak
- format: openai
- base_url: http://127.0.0.1:9/weak/v1
- api_key: test-key
-
-profiles:
- fast:
- type: passthrough
- target: weak
- smart-stage-router:
- type: stage_router
- capable: strong
- efficient: weak
- fallback_target_on_evict: strong
- picker: capable_first
- confidence_threshold: 0.7
-"""
_PYTHON_CONFIG = """
targets:
@@ -155,36 +120,9 @@ def _write(tmp_path: Path, text: str, name: str = "profiles.yaml") -> Path:
return path
-def test_dry_run_validates_rust_profiles_and_direct_targets(tmp_path: Path) -> None:
- # passthrough + stage_router + the two targets (directly addressable).
- path = _write(tmp_path, _RUST_CONFIG)
- # dry_run loads, resolves, and builds the registry, then returns without
- # binding a socket. No exception == a servable config.
- run_profile_server(str(path), dry_run=True)
-
-
-def test_dry_run_rejects_invalid_config(tmp_path: Path) -> None:
- path = _write(tmp_path, "profiles:\n bad:\n type: passthrough\n target: ghost\n")
- with pytest.raises(SwitchyardConfigError):
- run_profile_server(str(path), dry_run=True)
-
-
-def test_shipped_example_config_is_servable(monkeypatch: pytest.MonkeyPatch) -> None:
- # Guards examples/profiles.yaml from rotting; dry_run never connects upstream.
- monkeypatch.setenv("OPENROUTER_API_KEY", "dummy-key")
- example = Path(__file__).resolve().parents[1] / "examples" / "profiles.yaml"
- run_profile_server(str(example), dry_run=True)
-
-
-def test_python_profile_ids_classifies_config(tmp_path: Path) -> None:
- assert python_profile_ids(_write(tmp_path, _PYTHON_CONFIG)) == ["smart"]
- assert python_profile_ids(_write(tmp_path, _RUST_CONFIG, name="rust.yaml")) == []
-
-
-def test_serve_config_uses_fastapi_for_python_profiles(
+def test_serve_config_uses_fastapi_profile_table(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
- capsys: pytest.CaptureFixture[str],
) -> None:
path = _write(tmp_path, _PYTHON_CONFIG)
args = _serve_args(path)
@@ -211,10 +149,6 @@ def test_serve_config_uses_fastapi_for_python_profiles(
assert captured["models"] == ["smart", "weak", "provider/weak"]
assert captured["inbound_default"] == "both"
- stderr = capsys.readouterr().err
- assert "warning: Python-defined profile serving is deprecated." in stderr
- assert "Python FastAPI adapter" in stderr
- assert "Python profile(s): smart." in stderr
def test_profile_config_route_table_serves_mixed_profiles(
@@ -285,24 +219,6 @@ def test_profile_config_route_table_serves_mixed_profiles(
)
-def test_serve_config_fails_closed_when_python_profile_inspection_fails(
- monkeypatch: pytest.MonkeyPatch,
- tmp_path: Path,
-) -> None:
- path = _write(tmp_path, _RUST_CONFIG)
-
- def fail_inspection(_path: str) -> list[str]:
- raise RuntimeError("inspection exploded")
-
- monkeypatch.setattr(
- "switchyard.lib.profiles.loader.python_profile_ids",
- fail_inspection,
- )
-
- with pytest.raises(SystemExit, match="failed to inspect.*inspection exploded"):
- _cmd_serve_profile_config(_serve_args(path))
-
-
def _serve_args(path: Path) -> argparse.Namespace:
return argparse.Namespace(
config=str(path),