Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(server): load libsy algorithms from explicit TOML (#115)

nachiketb-nvidiaf47ab9b51b03
diff --git a/Cargo.lock b/Cargo.lock
index f45b55c22..9f7955951 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -430,6 +430,12 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "fastrand"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
+
[[package]]
name = "find-msvc-tools"
version = "0.1.9"
@@ -912,6 +918,12 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e126dda6f34391ab7b444f9922055facc83c07a910da3eb16f1e4d9c45dc777"
+[[package]]
+name = "linux-raw-sys"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
+
[[package]]
name = "litemap"
version = "0.8.2"
@@ -1397,6 +1409,19 @@ version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
+[[package]]
+name = "rustix"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
+dependencies = [
+ "bitflags",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "rustls"
version = "0.23.42"
@@ -1553,11 +1578,11 @@ dependencies = [
[[package]]
name = "serde_spanned"
-version = "0.6.9"
+version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
+checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26"
dependencies = [
- "serde",
+ "serde_core",
]
[[package]]
@@ -1764,11 +1789,14 @@ dependencies = [
"futures-util",
"http-body-util",
"rustls",
+ "serde",
"serde_json",
"switchyard-libsy",
"switchyard-llm-client",
"switchyard-translation",
+ "tempfile",
"tokio",
+ "toml",
"tower",
"tracing",
]
@@ -1834,6 +1862,19 @@ version = "0.13.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
+[[package]]
+name = "tempfile"
+version = "3.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
+dependencies = [
+ "fastrand",
+ "getrandom 0.4.3",
+ "once_cell",
+ "rustix",
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "thiserror"
version = "2.0.18"
@@ -1943,44 +1984,42 @@ dependencies = [
[[package]]
name = "toml"
-version = "0.8.23"
+version = "1.1.3+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
+checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c"
dependencies = [
- "serde",
+ "indexmap",
+ "serde_core",
"serde_spanned",
"toml_datetime",
- "toml_edit",
+ "toml_parser",
+ "toml_writer",
+ "winnow",
]
[[package]]
name = "toml_datetime"
-version = "0.6.11"
+version = "1.1.1+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
+checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7"
dependencies = [
- "serde",
+ "serde_core",
]
[[package]]
-name = "toml_edit"
-version = "0.22.27"
+name = "toml_parser"
+version = "1.1.2+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
+checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"
dependencies = [
- "indexmap",
- "serde",
- "serde_spanned",
- "toml_datetime",
- "toml_write",
"winnow",
]
[[package]]
-name = "toml_write"
-version = "0.1.2"
+name = "toml_writer"
+version = "1.1.2+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
+checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2"
[[package]]
name = "tower"
@@ -2283,12 +2322,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "winnow"
-version = "0.7.15"
+version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
-dependencies = [
- "memchr",
-]
+checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81"
[[package]]
name = "wiremock"
diff --git a/crates/switchyard-components-v2/Cargo.toml b/crates/switchyard-components-v2/Cargo.toml
index f5b78dd7e..204c2a2eb 100644
--- a/crates/switchyard-components-v2/Cargo.toml
+++ b/crates/switchyard-components-v2/Cargo.toml
@@ -23,7 +23,7 @@ serde_json = "1"
switchyard-components = { path = "../switchyard-components" }
switchyard-components-v2-macros = { path = "../switchyard-components-v2-macros" }
switchyard-core = { path = "../switchyard-core" }
-toml = "0.8"
+toml = "1.1"
tracing = { version = "0.1", default-features = false, features = ["std"] }
yaml_serde = "0.10"
diff --git a/crates/switchyard-server/CONFIGURATION.md b/crates/switchyard-server/CONFIGURATION.md
new file mode 100644
index 000000000..bc9e955c4
--- /dev/null
+++ b/crates/switchyard-server/CONFIGURATION.md
@@ -0,0 +1,26 @@
+# Extending Server Configuration
+
+## Add an LLM client and target
+
+Define the upstream once under `llm_clients`, then reference it from targets.
+
+```toml
+[llm_clients.provider]
+format = "openai_chat"
+base_url = "https://example.com/v1"
+api_key_env = "PROVIDER_API_KEY"
+
+[targets.model]
+id = "provider/model"
+llm_client = "provider"
+```
+
+To support another wire format, add its `ClientFormat` variant and explicit construction match in
+`src/config.rs`. Add a client type only when a second implementation exists.
+
+## Add an algorithm
+
+1. Implement and export the algorithm from `libsy`.
+2. Add its TOML fields as an `AlgorithmConfig` variant in `src/config.rs`.
+3. Construct it in the `build_algorithm` match, resolving target names with `resolve_targets`.
+4. Add a parsing test and an end-to-end server test when the algorithm makes LLM calls.
diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.toml
index 88e88ac02..1cbf9b1f4 100644
--- a/crates/switchyard-server/Cargo.toml
+++ b/crates/switchyard-server/Cargo.toml
@@ -19,6 +19,8 @@ rustls = { version = "0.23" }
clap = { version = "4", features = ["derive", "env"] }
futures-util = "0.3"
libsy = { package = "switchyard-libsy", path = "../libsy" }
+serde = { version = "1", features = ["derive"] }
+toml = "1.1"
switchyard-llm-client = { path = "../libsy-llm-client" }
serde_json = "1"
switchyard-translation = { path = "../switchyard-translation" }
@@ -28,5 +30,6 @@ tracing = { version = "0.1", default-features = false, features = ["std"] }
[dev-dependencies]
async-trait = "0.1"
http-body-util = "0.1"
+tempfile = "3"
tokio = { version = "1", features = ["macros", "rt", "net"] }
tower = { version = "0.5", features = ["util"] }
diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md
index 4bed0fd07..56838e013 100644
--- a/crates/switchyard-server/README.md
+++ b/crates/switchyard-server/README.md
@@ -1,15 +1,52 @@
# switchyard-server
`switchyard-server` exposes libsy algorithms through OpenAI Chat Completions, OpenAI Responses,
-and Anthropic Messages endpoints. The current CLI builds uniform-random routes over one shared
-upstream backend.
+and Anthropic Messages endpoints. A TOML file explicitly defines the LLM clients, targets, and
+algorithm routes served by the process.
+
+```toml
+# routes.toml
+schema_version = 1
+
+[llm_clients.example]
+format = "openai_chat"
+base_url = "https://example.com/v1"
+api_key_env = "API_KEY"
+
+[targets.model_a]
+id = "model/a"
+llm_client = "example"
+
+[targets.model_b]
+id = "model/b"
+llm_client = "example"
+
+[routes.general]
+id = "switchyard/general"
+type = "random"
+targets = ["model_a", "model_b"]
+
+[routes.classified]
+id = "switchyard/classified"
+type = "llm_classifier"
+classifier_target = "model_a"
+strong_target = "model_a"
+weak_target = "model_b"
+threshold = 0.5
+```
```bash
-cargo run -p switchyard-server -- \
- --route switchyard/general=model/a,model/b \
- --route switchyard/coding=model/c,model/d \
- --base-url https://example.com/v1 \
- --api-key "$API_KEY"
+export API_KEY="..."
+cargo run -p switchyard-server -- --config routes.toml
```
-Select a route by sending its `switchyard/*` ID as the request's `model`.
+Target and route table names are local references. A target's `id` is the exact model ID sent
+upstream, and a route's `id` is the model clients send to select that algorithm.
+
+Each target references an entry under `llm_clients`. All configured clients use
+`TranslatingLlmClient`; supported formats are `openai_chat`, `openai_responses`, and
+`anthropic_messages`. Supported algorithms are `noop`, `random`, and `llm_classifier`. An
+`api_key_env` value names an environment variable; the TOML never contains the secret itself. If
+omitted, the client sends no authentication.
+
+See [CONFIGURATION.md](CONFIGURATION.md) to add an LLM client, target, or algorithm.
diff --git a/crates/switchyard-server/src/cli.rs b/crates/switchyard-server/src/cli.rs
index 63e88345e..ac176f288 100644
--- a/crates/switchyard-server/src/cli.rs
+++ b/crates/switchyard-server/src/cli.rs
@@ -1,18 +1,13 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! CLI entrypoint for running the libsy server with random routing.
+//! CLI entrypoint for running the configured libsy server.
-use std::collections::{BTreeMap, BTreeSet};
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::path::PathBuf;
-use std::str::FromStr;
-use std::sync::Arc;
-use clap::{Parser, ValueEnum};
-use libsy::algorithms::Random;
-use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient};
-use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
+use clap::Parser;
+use switchyard_server::config::load_server_state;
use switchyard_server::{
run_server, ServerError, ServerResult, ServerRunOptions, ServerState, TlsOptions,
DEFAULT_LISTEN_BACKLOG,
@@ -21,88 +16,17 @@ use switchyard_server::{
const DEFAULT_HOST: IpAddr = IpAddr::V4(Ipv4Addr::UNSPECIFIED);
const DEFAULT_PORT: u16 = 4000;
-#[derive(Clone, Debug)]
-struct RandomRouteSpec {
- model: String,
- targets: Vec<String>,
-}
-
-impl FromStr for RandomRouteSpec {
- type Err = String;
-
- fn from_str(value: &str) -> Result<Self, Self::Err> {
- let (model, target_list) = value
- .split_once('=')
- .ok_or_else(|| "route must use MODEL=TARGET[,TARGET...]".to_string())?;
- let model = model.trim();
- if model.is_empty() {
- return Err("route model must not be empty".to_string());
- }
- let targets = target_list
- .split(',')
- .map(str::trim)
- .map(str::to_string)
- .collect::<Vec<_>>();
- if targets.iter().any(String::is_empty) {
- return Err(format!("route {model} must contain non-empty targets"));
- }
- if targets.iter().collect::<BTreeSet<_>>().len() != targets.len() {
- return Err(format!("route {model} must contain unique targets"));
- }
- Ok(Self {
- model: model.to_string(),
- targets,
- })
- }
-}
-
-#[derive(Clone, Copy, Debug, ValueEnum)]
-enum UpstreamFormat {
- #[value(name = "openai-chat")]
- OpenAiChat,
- #[value(name = "openai-responses")]
- OpenAiResponses,
- #[value(name = "anthropic")]
- Anthropic,
-}
-
-impl UpstreamFormat {
- fn backend(self, config: HttpBackendConfig) -> Backend {
- match self {
- Self::OpenAiChat => Backend::OpenAiChat(config),
- Self::OpenAiResponses => Backend::OpenAiResponses(config),
- Self::Anthropic => Backend::Anthropic(config),
- }
- }
-}
-
/// Command-line arguments accepted by the Rust server binary.
#[derive(Debug, Parser)]
#[command(
name = "switchyard-server",
- about = "Run uniform random routing with libsy",
+ about = "Serve explicitly configured libsy algorithms",
version
)]
pub(crate) struct ServerArgs {
- /// Random route as MODEL=TARGET[,TARGET...]. Repeat to serve multiple routes.
- #[arg(
- long = "route",
- required = true,
- value_name = "MODEL=TARGET[,TARGET...]"
- )]
- routes: Vec<RandomRouteSpec>,
-
- /// Base URL shared by the configured upstream targets.
- #[arg(long, env = "SWITCHYARD_UPSTREAM_BASE_URL")]
- base_url: String,
-
- /// Upstream API key. Omit when the backend needs no authentication.
- #[arg(long, env = "SWITCHYARD_UPSTREAM_API_KEY")]
- api_key: Option<String>,
-
- /// Provider wire format used by the upstream backend.
- #[arg(long, value_enum, default_value = "openai-chat")]
- upstream_format: UpstreamFormat,
+ /// TOML file defining LLM clients, targets, and algorithm routes.
+ #[arg(long, value_name = "PATH")]
+ config: PathBuf,
/// Host address to bind.
#[arg(long, default_value_t = DEFAULT_HOST)]
@@ -136,44 +60,7 @@ impl ServerArgs {
}
fn into_runtime(self) -> ServerResult<(ServerState, ServerRunOptions)> {
- if self.base_url.trim().is_empty() {
- return Err(ServerError::new("--base-url must not be empty"));
- }
-
- let backend = self.upstream_format.backend(HttpBackendConfig {
- base_url: self.base_url,
- api_key: self.api_key,
- extra_headers: BTreeMap::new(),
- });
- let target_models = self
- .routes
- .iter()
- .flat_map(|route| route.targets.iter())
- .collect::<BTreeSet<_>>();
- let model_configs = target_models
- .into_iter()
- .map(|model| ModelConfig::new(model.as_str(), backend.clone(), None))
- .collect::<Vec<_>>();
- let client: Arc<dyn RoutedLlmClient> = Arc::new(
- TranslatingLlmClient::new(&model_configs)
- .map_err(|error| ServerError::new(error.to_string()))?,
- );
- let routes = self.routes.into_iter().map(|route| {
- let target_set = LlmTargetSet::new(
- route
- .targets
- .iter()
- .map(|model| LlmTarget {
- semantic_name: model.clone(),
- llm_client: Some(Arc::clone(&client)),
- })
- .collect(),
- );
- let algorithm: Arc<dyn Algorithm> = Arc::new(Random::new(target_set));
- (route.model, algorithm)
- });
- let state = ServerState::new(routes)?;
-
+ let state = load_server_state(&self.config)?;
let tls = match (self.tls_cert, self.tls_key) {
(Some(cert), Some(key)) => {
if !cert.exists() || !key.exists() {
@@ -197,46 +84,8 @@ impl ServerArgs {
}
}
-/// Builds the random algorithm and starts the server.
+/// Loads the configured algorithms and starts the server.
pub(crate) async fn run(args: ServerArgs) -> ServerResult<()> {
let (state, options) = args.into_runtime()?;
run_server(state, options).await
}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- fn runtime(routes: &[&str]) -> ServerResult<ServerState> {
- let mut args = vec!["switchyard-server", "--base-url", "http://127.0.0.1:9/v1"];
- for route in routes {
- args.extend(["--route", route]);
- }
- let args = ServerArgs::try_parse_from(args)
- .map_err(|error| ServerError::new(error.to_string()))?;
- args.into_runtime().map(|(state, _)| state)
- }
-
- #[test]
- fn routes_build_independent_algorithms() -> ServerResult<()> {
- let state = runtime(&[
- "switchyard/general=model/a,model/b",
- "switchyard/coding=model/c,model/d",
- ])?;
- assert_eq!(
- state.models().collect::<Vec<_>>(),
- ["switchyard/coding", "switchyard/general"]
- );
- Ok(())
- }
-
- #[test]
- fn invalid_routes_are_rejected() {
- for routes in [
- &["switchyard/general=model/a,model/a"][..],
- &["switchyard/general=model/a", "switchyard/general=model/b"],
- ] {
- assert!(runtime(routes).is_err());
- }
- }
-}
diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs
new file mode 100644
index 000000000..ded8d4695
--- /dev/null
+++ b/crates/switchyard-server/src/config.rs
@@ -0,0 +1,459 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Typed TOML configuration and explicit construction for the Rust server.
+
+use std::collections::{BTreeMap, BTreeSet};
+use std::fs;
+use std::path::Path;
+use std::sync::Arc;
+
+use libsy::algorithms::{LlmClassifier, Noop, Random};
+use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient};
+use serde::Deserialize;
+use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
+
+use crate::{ServerError, ServerResult, ServerState};
+
+const SUPPORTED_SCHEMA_VERSION: u32 = 1;
+
+/// Loads a TOML deployment file and constructs the complete server state.
+pub fn load_server_state(path: impl AsRef<Path>) -> ServerResult<ServerState> {
+ let path = path.as_ref();
+ let toml = fs::read_to_string(path).map_err(|error| {
+ ServerError::new(format!(
+ "failed to read server config {}: {error}",
+ path.display()
+ ))
+ })?;
+ server_state_from_toml(&toml).map_err(|error| {
+ ServerError::new(format!("invalid server config {}: {error}", path.display()))
+ })
+}
+
+fn server_state_from_toml(toml: &str) -> ServerResult<ServerState> {
+ let config: ServerConfig = toml::from_str(toml)
+ .map_err(|error| ServerError::new(format!("failed to parse TOML: {error}")))?;
+ config.build()
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct ServerConfig {
+ schema_version: u32,
+ #[serde(default)]
+ llm_clients: BTreeMap<String, LlmClientConfig>,
+ targets: BTreeMap<String, TargetConfig>,
+ routes: BTreeMap<String, RouteConfig>,
+}
+
+impl ServerConfig {
+ fn build(&self) -> ServerResult<ServerState> {
+ if self.schema_version != SUPPORTED_SCHEMA_VERSION {
+ return Err(ServerError::new(format!(
+ "unsupported schema_version {}; expected {SUPPORTED_SCHEMA_VERSION}",
+ self.schema_version
+ )));
+ }
+
+ let clients = self.build_clients()?;
+ let targets = self.build_targets(&clients)?;
+ let mut routes = Vec::with_capacity(self.routes.len());
+ for (route_name, config) in &self.routes {
+ validate_value("route name", route_name)?;
+ validate_value(&format!("route {route_name} id"), config.id())?;
+ routes.push((
+ config.id().to_string(),
+ build_algorithm(route_name, config, &targets)?,
+ ));
+ }
+ ServerState::new(routes)
+ }
+
+ fn build_clients(&self) -> ServerResult<BTreeMap<String, Arc<dyn RoutedLlmClient>>> {
+ let mut models_by_client = self
+ .llm_clients
+ .keys()
+ .map(|name| (name.clone(), Vec::new()))
+ .collect::<BTreeMap<String, Vec<ModelConfig>>>();
+
+ for name in self.llm_clients.keys() {
+ validate_value("llm client name", name)?;
+ }
+ for (target_name, target) in &self.targets {
+ validate_value("target name", target_name)?;
+ validate_value(&format!("target {target_name} id"), &target.id)?;
+ let client_config = self.llm_clients.get(&target.llm_client).ok_or_else(|| {
+ ServerError::new(format!(
+ "target {target_name} references unknown llm client {}",
+ target.llm_client
+ ))
+ })?;
+ let model_configs = models_by_client
+ .get_mut(&target.llm_client)
+ .ok_or_else(|| ServerError::new("validated llm client was not initialized"))?;
+ model_configs.push(ModelConfig::new(
+ &target.id,
+ build_backend(&target.llm_client, client_config)?,
+ None,
+ ));
+ }
+
+ let mut clients = BTreeMap::new();
+ for (name, model_configs) in models_by_client {
+ let client: Arc<dyn RoutedLlmClient> = Arc::new(
+ TranslatingLlmClient::new(&model_configs)
+ .map_err(|error| ServerError::new(error.to_string()))?,
+ );
+ clients.insert(name, client);
+ }
+ Ok(clients)
+ }
+
+ fn build_targets(
+ &self,
+ clients: &BTreeMap<String, Arc<dyn RoutedLlmClient>>,
+ ) -> ServerResult<BTreeMap<String, LlmTarget>> {
+ self.targets
+ .iter()
+ .map(|(name, config)| {
+ let client = clients.get(&config.llm_client).ok_or_else(|| {
+ ServerError::new(format!("target {name} has no constructed llm client"))
+ })?;
+ Ok((
+ name.clone(),
+ LlmTarget {
+ semantic_name: config.id.clone(),
+ llm_client: Some(Arc::clone(client)),
+ },
+ ))
+ })
+ .collect()
+ }
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct LlmClientConfig {
+ format: ClientFormat,
+ base_url: String,
+ api_key_env: Option<String>,
+ #[serde(default)]
+ extra_headers: BTreeMap<String, String>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct TargetConfig {
+ id: String,
+ llm_client: String,
+}
+
+#[derive(Clone, Copy, Debug, Deserialize)]
+enum ClientFormat {
+ #[serde(rename = "openai_chat")]
+ OpenAiChat,
+ #[serde(rename = "openai_responses")]
+ OpenAiResponses,
+ #[serde(rename = "anthropic_messages")]
+ AnthropicMessages,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
+enum RouteConfig {
+ Noop {
+ id: String,
+ },
+ Random {
+ id: String,
+ targets: Vec<String>,
+ },
+ LlmClassifier {
+ id: String,
+ classifier_target: String,
+ strong_target: String,
+ weak_target: String,
+ threshold: f64,
+ },
+}
+
+impl RouteConfig {
+ fn id(&self) -> &str {
+ match self {
+ Self::Noop { id } | Self::Random { id, .. } | Self::LlmClassifier { id, .. } => id,
+ }
+ }
+}
+
+fn build_backend(client_name: &str, config: &LlmClientConfig) -> ServerResult<Backend> {
+ let base_url = config.base_url.trim();
+ if base_url.is_empty() {
+ return Err(ServerError::new(format!(
+ "llm client {client_name} base_url must not be empty"
+ )));
+ }
+ let api_key = config
+ .api_key_env
+ .as_deref()
+ .map(|variable| {
+ if variable.trim().is_empty() {
+ return Err(ServerError::new(format!(
+ "llm client {client_name} api_key_env must not be empty"
+ )));
+ }
+ let api_key = std::env::var(variable).map_err(|error| {
+ ServerError::new(format!(
+ "llm client {client_name} could not read api_key_env {variable}: {error}"
+ ))
+ })?;
+ if api_key.trim().is_empty() {
+ return Err(ServerError::new(format!(
+ "llm client {client_name} api_key_env {variable} is empty"
+ )));
+ }
+ Ok(api_key)
+ })
+ .transpose()?;
+ let http = HttpBackendConfig {
+ base_url: base_url.to_string(),
+ api_key,
+ extra_headers: config.extra_headers.clone(),
+ };
+ Ok(match config.format {
+ ClientFormat::OpenAiChat => Backend::OpenAiChat(http),
+ ClientFormat::OpenAiResponses => Backend::OpenAiResponses(http),
+ ClientFormat::AnthropicMessages => Backend::Anthropic(http),
+ })
+}
+
+fn build_algorithm(
+ route_name: &str,
+ config: &RouteConfig,
+ targets: &BTreeMap<String, LlmTarget>,
+) -> ServerResult<Arc<dyn Algorithm>> {
+ match config {
+ RouteConfig::Noop { .. } => Ok(Arc::new(Noop {})),
+ RouteConfig::Random { targets: names, .. } => {
+ if names.is_empty() {
+ return Err(ServerError::new(format!(
+ "random route {route_name} requires at least one target"
+ )));
+ }
+ let unique = names.iter().collect::<BTreeSet<_>>();
+ if unique.len() != names.len() {
+ return Err(ServerError::new(format!(
+ "random route {route_name} contains duplicate targets"
+ )));
+ }
+ Ok(Arc::new(Random::new(resolve_targets(
+ route_name,
+ names.iter().map(String::as_str),
+ targets,
+ )?)))
+ }
+ RouteConfig::LlmClassifier {
+ classifier_target,
+ strong_target,
+ weak_target,
+ threshold,
+ ..
+ } => {
+ if !threshold.is_finite() || !(0.0..=1.0).contains(threshold) {
+ return Err(ServerError::new(format!(
+ "llm_classifier route {route_name} threshold must be between 0 and 1"
+ )));
+ }
+ let classifier = resolve_target(route_name, classifier_target, targets)?;
+ let strong = resolve_target(route_name, strong_target, targets)?;
+ let weak = resolve_target(route_name, weak_target, targets)?;
+ let target_set =
+ LlmTargetSet::new(vec![classifier.clone(), strong.clone(), weak.clone()]);
+ Ok(Arc::new(LlmClassifier::new(
+ classifier.semantic_name,
+ strong.semantic_name,
+ weak.semantic_name,
+ *threshold,
+ target_set,
+ )))
+ }
+ }
+}
+
+fn resolve_targets<'a>(
+ route_name: &str,
+ names: impl IntoIterator<Item = &'a str>,
+ targets: &BTreeMap<String, LlmTarget>,
+) -> ServerResult<LlmTargetSet> {
+ let resolved = names
+ .into_iter()
+ .map(|name| resolve_target(route_name, name, targets))
+ .collect::<ServerResult<Vec<_>>>()?;
+ Ok(LlmTargetSet::new(resolved))
+}
+
+fn resolve_target(
+ route_name: &str,
+ name: &str,
+ targets: &BTreeMap<String, LlmTarget>,
+) -> ServerResult<LlmTarget> {
+ targets.get(name).cloned().ok_or_else(|| {
+ ServerError::new(format!(
+ "route {route_name} references unknown target {name}"
+ ))
+ })
+}
+
+fn validate_value(label: &str, value: &str) -> ServerResult<()> {
+ if value.trim().is_empty() || value.trim() != value {
+ return Err(ServerError::new(format!(
+ "{label} must be non-empty and have no surrounding whitespace"
+ )));
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const VALID_CONFIG: &str = r#"
+schema_version = 1
+
+[llm_clients.primary]
+format = "openai_chat"
+base_url = "https://example.test/v1"
+
+[llm_clients.responses]
+format = "openai_responses"
+base_url = "https://example.test/v1"
+
+[llm_clients.anthropic]
+format = "anthropic_messages"
+base_url = "https://example.test"
+
+[targets.classifier]
+id = "classifier/model"
+llm_client = "primary"
+
+[targets.strong]
+id = "strong/model"
+llm_client = "responses"
+
+[targets.weak]
+id = "weak/model"
+llm_client = "anthropic"
+
+[routes.noop]
+id = "switchyard/noop"
+type = "noop"
+
+[routes.random]
+id = "switchyard/random"
+type = "random"
+targets = ["strong", "weak"]
+
+[routes.classifier]
+id = "switchyard/classifier"
+type = "llm_classifier"
+classifier_target = "classifier"
+strong_target = "strong"
+weak_target = "weak"
+threshold = 0.5
+"#;
+
+ fn error_message(toml: &str) -> String {
+ match server_state_from_toml(toml) {
+ Ok(_) => "configuration unexpectedly succeeded".to_string(),
+ Err(error) => error.to_string(),
+ }
+ }
+
+ #[test]
+ fn builds_all_supported_algorithm_types() -> ServerResult<()> {
+ let state = server_state_from_toml(VALID_CONFIG)?;
+ assert_eq!(
+ state.models().collect::<Vec<_>>(),
+ [
+ "switchyard/classifier",
+ "switchyard/noop",
+ "switchyard/random"
+ ]
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn rejects_unknown_fields_and_algorithm_types() {
+ let unknown_field =
+ VALID_CONFIG.replace("schema_version = 1", "schema_version = 1\nmagic = true");
+ assert!(error_message(&unknown_field).contains("unknown field"));
+
+ let unknown_algorithm = VALID_CONFIG.replace("type = \"noop\"", "type = \"imaginary\"");
+ assert!(error_message(&unknown_algorithm).contains("unknown variant"));
+ }
+
+ #[test]
+ fn rejects_invalid_references_and_parameters() {
+ let cases = [
+ (
+ VALID_CONFIG.replace("llm_client = \"primary\"", "llm_client = \"missing\""),
+ "unknown llm client missing",
+ ),
+ (
+ VALID_CONFIG.replace(
+ "targets = [\"strong\", \"weak\"]",
+ "targets = [\"strong\", \"missing\"]",
+ ),
+ "unknown target missing",
+ ),
+ (
+ VALID_CONFIG.replace(
+ "targets = [\"strong\", \"weak\"]",
+ "targets = [\"strong\", \"strong\"]",
+ ),
+ "duplicate targets",
+ ),
+ (
+ VALID_CONFIG.replace("threshold = 0.5", "threshold = 1.5"),
+ "threshold must be between 0 and 1",
+ ),
+ (
+ VALID_CONFIG.replace("schema_version = 1", "schema_version = 2"),
+ "unsupported schema_version 2",
+ ),
+ (
+ VALID_CONFIG.replace("[targets.strong]", "[targets.\" strong \"]"),
+ "target name must be non-empty and have no surrounding whitespace",
+ ),
+ ];
+
+ for (toml, expected) in cases {
+ assert!(
+ error_message(&toml).contains(expected),
+ "expected error containing {expected}"
+ );
+ }
+ }
+
+ #[test]
+ fn api_key_environment_reference_is_validated() {
+ let missing = VALID_CONFIG.replacen(
+ "base_url = \"https://example.test/v1\"",
+ "base_url = \"https://example.test/v1\"\napi_key_env = \"SWITCHYARD_CONFIG_TEST_KEY_THAT_IS_NOT_SET\"",
+ 1,
+ );
+ assert!(error_message(&missing).contains("SWITCHYARD_CONFIG_TEST_KEY_THAT_IS_NOT_SET"));
+
+ const EMPTY_KEY_ENV: &str = "SWITCHYARD_CONFIG_TEST_EMPTY_KEY";
+ std::env::set_var(EMPTY_KEY_ENV, "");
+ let empty = VALID_CONFIG.replacen(
+ "base_url = \"https://example.test/v1\"",
+ &format!("base_url = \"https://example.test/v1\"\napi_key_env = \"{EMPTY_KEY_ENV}\""),
+ 1,
+ );
+ let message = error_message(&empty);
+ std::env::remove_var(EMPTY_KEY_ENV);
+ assert!(message.contains("is empty"));
+ }
+}
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index a49e4e354..192e9eec5 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -3,6 +3,7 @@
//! Rust HTTP server for libsy algorithms.
+pub mod config;
mod response;
mod sse;
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 0db025d20..b4698b204 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -6,6 +6,7 @@
use std::collections::{BTreeMap, HashSet};
use std::convert::Infallible;
use std::error::Error;
+use std::io::Write;
use std::sync::Arc;
use axum::body::{Body, Bytes};
@@ -20,6 +21,7 @@ use libsy::algorithms::Random;
use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient};
use serde_json::{json, Value};
use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
+use switchyard_server::config::load_server_state;
use switchyard_server::{build_switchyard_router, ServerState};
use tokio::net::TcpListener;
use tokio::sync::Mutex;
@@ -93,13 +95,18 @@ async fn upstream_chat(
return Sse::new(stream).into_response();
}
+ let content = if model == "model/classifier" {
+ "0.9"
+ } else {
+ "ok"
+ };
Json(json!({
"id": "chatcmpl-test",
"object": "chat.completion",
"model": model,
"choices": [{
"index": 0,
- "message": {"role": "assistant", "content": "ok"},
+ "message": {"role": "assistant", "content": content},
"finish_reason": "stop"
}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}
@@ -144,6 +151,16 @@ async fn test_app(routes: &[(&str, &[&str])]) -> TestResult<(MockUpstream, Route
Ok((upstream, app))
}
+fn load_test_config(toml: &str) -> TestResult<ServerState> {
+ let mut config = tempfile::Builder::new()
+ .prefix("switchyard-server-config-")
+ .suffix(".toml")
+ .tempfile()?;
+ config.write_all(toml.as_bytes())?;
+ config.flush()?;
+ Ok(load_server_state(config.path())?)
+}
+
async fn send(app: &Router, method: &str, path: &str, body: Option<Value>) -> TestResult<Response> {
let mut builder = HttpRequest::builder().method(method).uri(path);
let request_body = if let Some(body) = body {
@@ -179,6 +196,78 @@ impl Response {
}
}
+#[tokio::test]
+async fn toml_config_constructs_and_serves_multiple_algorithms() -> TestResult {
+ let upstream = MockUpstream::start().await?;
+ let state = load_test_config(&format!(
+ r#"
+schema_version = 1
+
+[llm_clients.upstream]
+format = "openai_chat"
+base_url = "{base_url}"
+
+[targets.classifier]
+id = "model/classifier"
+llm_client = "upstream"
+
+[targets.strong]
+id = "model/strong"
+llm_client = "upstream"
+
+[targets.weak]
+id = "model/weak"
+llm_client = "upstream"
+
+[routes.random]
+id = "switchyard/random"
+type = "random"
+targets = ["weak"]
+
+[routes.classifier]
+id = "switchyard/classifier"
+type = "llm_classifier"
+classifier_target = "classifier"
+strong_target = "strong"
+weak_target = "weak"
+threshold = 0.5
+"#,
+ base_url = upstream.base_url
+ ))?;
+ let app = build_switchyard_router(state);
+
+ for (route, selected) in [
+ ("switchyard/random", "model/weak"),
+ ("switchyard/classifier", "model/strong"),
+ ] {
+ let response = send(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({
+ "model": route,
+ "messages": [{"role": "user", "content": "hi"}]
+ })),
+ )
+ .await?;
+ assert_eq!(response.status, StatusCode::OK);
+ assert_eq!(
+ response
+ .headers
+ .get("x-model-router-selected-model")
+ .and_then(|value| value.to_str().ok()),
+ Some(selected)
+ );
+ }
+
+ let calls = upstream.calls.lock().await;
+ assert_eq!(calls.len(), 3);
+ assert_eq!(calls[0]["model"], "model/weak");
+ assert_eq!(calls[1]["model"], "model/classifier");
+ assert_eq!(calls[2]["model"], "model/strong");
+ Ok(())
+}
+
#[tokio::test]
async fn routes_dispatch_and_discovery_endpoints_are_stable() -> TestResult {
let (upstream, app) = test_app(&[