Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
mod config;
mod runtime;
mod span_capture;
mod translation;
use std::sync::Arc;
use nemo_relay_plugin::{ConfigDiagnostic, DiagnosticLevel, Json, NativePlugin, PluginContext};
use serde_json::Map;
use crate::config::{SwitchyardConfig, protocol_from_call};
use crate::runtime::{
RoutingEvent, RoutingEventEmitter, SwitchyardRuntime, emit_event as emit_routing_event,
emit_events,
};
use crate::span_capture::SpanCapture;
#[derive(Default)]
struct SwitchyardPlugin;
impl NativePlugin for SwitchyardPlugin {
fn plugin_kind(&self) -> &str {
"nvidia.switchyard"
}
fn allows_multiple_components(&self) -> bool {
false
}
fn validate(&self, plugin_config: &Map<String, Json>) -> Vec<ConfigDiagnostic> {
// Allow `SwitchyardRuntime::new` to run for accurate validation diagnostics.
match parse_config(plugin_config).and_then(SwitchyardRuntime::new) {
Ok(_) => Vec::new(),
Err(message) => vec![ConfigDiagnostic {
level: DiagnosticLevel::Error,
code: "switchyard.invalid_config".into(),
component: Some("nvidia.switchyard".into()),
field: Some("config".into()),
message,
}],
}
}
fn register(
&mut self,
plugin_config: &Map<String, Json>,
ctx: &mut PluginContext<'_>,
) -> nemo_relay_plugin::Result<()> {
let config = parse_config(plugin_config)?;
let payload_limit = config
.capture_routing_payloads
.then_some(config.max_routing_payload_bytes);
let priority = config.priority;
let runtime = Arc::new(SwitchyardRuntime::new(config)?);
let plugin_runtime = ctx.runtime();
let event_runtime = Arc::clone(&runtime);
let emit_event: RoutingEventEmitter = Arc::new(move |event| {
emit_routing_event(&plugin_runtime, event_runtime.redactor(), event)
});
let span_runtime = Arc::clone(&runtime);
let span_emit_event = Arc::clone(&emit_event);
let span_capture = SpanCapture::new(payload_limit, move |mut span| {
span.sanitize(span_runtime.redactor());
span_emit_event(RoutingEvent::Span(span));
});
register_buffered(
ctx,
priority,
Arc::clone(&runtime),
Arc::clone(&emit_event),
span_capture.clone(),
)?;
register_stream(ctx, priority, runtime, emit_event, span_capture)?;
Ok(())
}
}
fn register_buffered(
ctx: &mut PluginContext<'_>,
priority: i32,
runtime: Arc<SwitchyardRuntime>,
emit_event: RoutingEventEmitter,
span_capture: SpanCapture,
) -> Result<(), String> {
ctx.register_llm_execution_intercept(
"switchyard.runner.buffered",
priority,
move |name, request, _context, next| {
let runtime = Arc::clone(&runtime);
let emit_event = Arc::clone(&emit_event);
let span_capture = span_capture.clone();
async move {
let Some(inbound) = protocol_from_call(&name) else {
return next.call(request).await;
};
if !request_model(&request).is_some_and(|model| runtime.manages_model(model)) {
return next.call(request).await;
}
let decoded = runtime.decode_request(inbound, request, false)?;
let execution = span_capture
.capture(runtime.execute_buffered(inbound, decoded))
.await;
emit_events(&emit_event, execution.events);
execution.result
}
},
)
}
fn register_stream(
ctx: &mut PluginContext<'_>,
priority: i32,
runtime: Arc<SwitchyardRuntime>,
emit_event: RoutingEventEmitter,
span_capture: SpanCapture,
) -> Result<(), String> {
ctx.register_llm_stream_execution_intercept(
"switchyard.runner.streaming",
priority,
move |name, request, _context, next| {
let runtime = Arc::clone(&runtime);
let emit_event = Arc::clone(&emit_event);
let span_capture = span_capture.clone();
async move {
let Some(inbound) = protocol_from_call(&name) else {
return next.call(request).await;
};
if !request_model(&request).is_some_and(|model| runtime.manages_model(model)) {
return next.call(request).await;
}
let decoded = runtime.decode_request(inbound, request, true)?;
let execution = span_capture
.capture(runtime.execute_stream(inbound, decoded, Arc::clone(&emit_event)))
.await;
emit_events(&emit_event, execution.events);
execution.result
}
},
)
}
fn parse_config(plugin_config: &Map<String, Json>) -> Result<SwitchyardConfig, String> {
let mut config = plugin_config.clone();
config.remove("executor");
serde_json::from_value(Json::Object(config))
.map_err(|error| format!("invalid Switchyard configuration: {error}"))
}
fn request_model(request: &nemo_relay_plugin::LlmRequest) -> Option<&str> {
request.content.get("model").and_then(Json::as_str)
}
nemo_relay_plugin::nemo_relay_plugin!(nemo_relay_register_plugin, SwitchyardPlugin::default);
#[cfg(test)]
mod tests {
use serde_json::json;
use super::*;
#[test]
fn plugin_configuration_requires_a_switchyard_source() {
let config = json!({"priority": 0});
assert!(matches!(
parse_config(config.as_object().unwrap()).and_then(SwitchyardRuntime::new),
Err(error) if error.contains("switchyard_config_path or switchyard_config")
));
}
#[test]
fn plugin_configuration_ignores_the_sdk_executor_override() {
let config = json!({
"executor": {"worker_threads": 4},
"switchyard_config_path": "routes.toml"
});
assert!(parse_config(config.as_object().unwrap()).is_ok());
}
#[test]
fn request_model_reads_only_a_top_level_string() {
let request = nemo_relay_plugin::LlmRequest {
headers: Map::new(),
content: json!({"model": "switchyard/default", "messages": []}),
};
assert_eq!(request_model(&request), Some("switchyard/default"));
let request = nemo_relay_plugin::LlmRequest {
headers: Map::new(),
content: json!({"model": {"name": "switchyard/default"}}),
};
assert_eq!(request_model(&request), None);
}
}