Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
6.7 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

mod config;
mod runtime;
mod span_capture;
mod translation;

use std::sync::Arc;

use nemo_relay_plugin::{ConfigDiagnostic, DiagnosticLevel, Json, NativePlugin, PluginContext};
use serde_json::Map;

use crate::config::{SwitchyardConfig, protocol_from_call};
use crate::runtime::{
    RoutingEvent, RoutingEventEmitter, SwitchyardRuntime, emit_event as emit_routing_event,
    emit_events,
};
use crate::span_capture::SpanCapture;

#[derive(Default)]
struct SwitchyardPlugin;

impl NativePlugin for SwitchyardPlugin {
    fn plugin_kind(&self) -> &str {
        "nvidia.switchyard"
    }

    fn allows_multiple_components(&self) -> bool {
        false
    }

    fn validate(&self, plugin_config: &Map<String, Json>) -> Vec<ConfigDiagnostic> {
        // Allow `SwitchyardRuntime::new` to run for accurate validation diagnostics.
        match parse_config(plugin_config).and_then(SwitchyardRuntime::new) {
            Ok(_) => Vec::new(),
            Err(message) => vec![ConfigDiagnostic {
                level: DiagnosticLevel::Error,
                code: "switchyard.invalid_config".into(),
                component: Some("nvidia.switchyard".into()),
                field: Some("config".into()),
                message,
            }],
        }
    }

    fn register(
        &mut self,
        plugin_config: &Map<String, Json>,
        ctx: &mut PluginContext<'_>,
    ) -> nemo_relay_plugin::Result<()> {
        let config = parse_config(plugin_config)?;
        let payload_limit = config
            .capture_routing_payloads
            .then_some(config.max_routing_payload_bytes);
        let priority = config.priority;
        let runtime = Arc::new(SwitchyardRuntime::new(config)?);
        let plugin_runtime = ctx.runtime();
        let event_runtime = Arc::clone(&runtime);
        let emit_event: RoutingEventEmitter = Arc::new(move |event| {
            emit_routing_event(&plugin_runtime, event_runtime.redactor(), event)
        });
        let span_runtime = Arc::clone(&runtime);
        let span_emit_event = Arc::clone(&emit_event);
        let span_capture = SpanCapture::new(payload_limit, move |mut span| {
            span.sanitize(span_runtime.redactor());
            span_emit_event(RoutingEvent::Span(span));
        });
        register_buffered(
            ctx,
            priority,
            Arc::clone(&runtime),
            Arc::clone(&emit_event),
            span_capture.clone(),
        )?;
        register_stream(ctx, priority, runtime, emit_event, span_capture)?;
        Ok(())
    }
}

fn register_buffered(
    ctx: &mut PluginContext<'_>,
    priority: i32,
    runtime: Arc<SwitchyardRuntime>,
    emit_event: RoutingEventEmitter,
    span_capture: SpanCapture,
) -> Result<(), String> {
    ctx.register_llm_execution_intercept(
        "switchyard.runner.buffered",
        priority,
        move |name, request, _context, next| {
            let runtime = Arc::clone(&runtime);
            let emit_event = Arc::clone(&emit_event);
            let span_capture = span_capture.clone();
            async move {
                let Some(inbound) = protocol_from_call(&name) else {
                    return next.call(request).await;
                };
                if !request_model(&request).is_some_and(|model| runtime.manages_model(model)) {
                    return next.call(request).await;
                }
                let decoded = runtime.decode_request(inbound, request, false)?;
                let execution = span_capture
                    .capture(runtime.execute_buffered(inbound, decoded))
                    .await;
                emit_events(&emit_event, execution.events);
                execution.result
            }
        },
    )
}

fn register_stream(
    ctx: &mut PluginContext<'_>,
    priority: i32,
    runtime: Arc<SwitchyardRuntime>,
    emit_event: RoutingEventEmitter,
    span_capture: SpanCapture,
) -> Result<(), String> {
    ctx.register_llm_stream_execution_intercept(
        "switchyard.runner.streaming",
        priority,
        move |name, request, _context, next| {
            let runtime = Arc::clone(&runtime);
            let emit_event = Arc::clone(&emit_event);
            let span_capture = span_capture.clone();
            async move {
                let Some(inbound) = protocol_from_call(&name) else {
                    return next.call(request).await;
                };
                if !request_model(&request).is_some_and(|model| runtime.manages_model(model)) {
                    return next.call(request).await;
                }
                let decoded = runtime.decode_request(inbound, request, true)?;
                let execution = span_capture
                    .capture(runtime.execute_stream(inbound, decoded, Arc::clone(&emit_event)))
                    .await;
                emit_events(&emit_event, execution.events);
                execution.result
            }
        },
    )
}

fn parse_config(plugin_config: &Map<String, Json>) -> Result<SwitchyardConfig, String> {
    let mut config = plugin_config.clone();
    config.remove("executor");
    serde_json::from_value(Json::Object(config))
        .map_err(|error| format!("invalid Switchyard configuration: {error}"))
}

fn request_model(request: &nemo_relay_plugin::LlmRequest) -> Option<&str> {
    request.content.get("model").and_then(Json::as_str)
}

nemo_relay_plugin::nemo_relay_plugin!(nemo_relay_register_plugin, SwitchyardPlugin::default);

#[cfg(test)]
mod tests {
    use serde_json::json;

    use super::*;

    #[test]
    fn plugin_configuration_requires_a_switchyard_source() {
        let config = json!({"priority": 0});
        assert!(matches!(
            parse_config(config.as_object().unwrap()).and_then(SwitchyardRuntime::new),
            Err(error) if error.contains("switchyard_config_path or switchyard_config")
        ));
    }

    #[test]
    fn plugin_configuration_ignores_the_sdk_executor_override() {
        let config = json!({
            "executor": {"worker_threads": 4},
            "switchyard_config_path": "routes.toml"
        });
        assert!(parse_config(config.as_object().unwrap()).is_ok());
    }

    #[test]
    fn request_model_reads_only_a_top_level_string() {
        let request = nemo_relay_plugin::LlmRequest {
            headers: Map::new(),
            content: json!({"model": "switchyard/default", "messages": []}),
        };
        assert_eq!(request_model(&request), Some("switchyard/default"));

        let request = nemo_relay_plugin::LlmRequest {
            headers: Map::new(),
            content: json!({"model": {"name": "switchyard/default"}}),
        };
        assert_eq!(request_model(&request), None);
    }
}