Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
9.9 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::{
    collections::{BTreeMap, HashMap},
    path::Path,
    sync::Mutex,
    time::{Duration, Instant},
};

use anyhow::{Context, Result, ensure};
use serde_json::value::{RawValue, to_raw_value};
use switchyard_protocol::ModelId;
use switchyard_runner::Runner;

use crate::request;

pub const MODEL_HEADER: &str = "x-gateway-model-name";
// Match SDK 0.3's idle TTL; its retained state is reclaimed by an hourly sweep.
const SESSION_TTL: Duration = Duration::from_secs(60 * 60);
type SessionIdentity = (String, String, Option<String>);

#[derive(serde::Deserialize)]
struct RequestModel {
    model: String,
}

pub struct Router {
    runner: Runner,
    sessions: Mutex<HashMap<SessionIdentity, Instant>>,
    pub(crate) metrics: Option<crate::metrics::Metrics>,
}

impl Router {
    pub fn load(routes: impl AsRef<Path>) -> Result<Self> {
        // SDK parse/build access would allow validation of decision-only options before construction.
        let mut router = Self::new(Runner::load(routes)?);
        router.metrics = std::env::var("METRICS_CONFIG")
            .ok()
            .map(|path| crate::metrics::Metrics::load(Path::new(&path)))
            .transpose()?;
        Ok(router)
    }

    pub fn new(runner: Runner) -> Self {
        Self {
            runner,
            sessions: Mutex::new(HashMap::new()),
            metrics: None,
        }
    }

    pub fn is_passthrough(&self, body: &[u8]) -> bool {
        serde_json::from_slice::<RequestModel>(body)
            .ok()
            .and_then(|request| self.runner.route(&request.model))
            .is_some_and(|route| route.algorithm_name() == "passthrough")
    }

    pub async fn decide(
        &self,
        body: &[u8],
        headers: &http::HeaderMap,
    ) -> Result<(Vec<u8>, String, Option<crate::metrics::InFlight>)> {
        let started = Instant::now();
        let request = {
            let raw = serde_json::from_slice(body)?;
            request::decode(&raw, headers)?
        };
        let input_model = request
            .llm_request
            .model
            .as_ref()
            .context("missing request model")?
            .as_str();
        let route_id = ModelId::from(input_model);
        let route = self
            .runner
            .route(route_id.as_str())
            .context("model is outside the configured catalog")?;
        let agent = request.metadata.as_ref().and_then(|m| m.agent_id.clone());
        ensure!(
            agent.as_ref().is_none_or(|id| id.len() <= 256),
            "agent id exceeds 256 bytes"
        );
        let is_session_final =
            request.metadata.as_ref().and_then(|m| m.session_final) == Some(true);
        let admission = if let Some(session) = request
            .metadata
            .as_ref()
            .and_then(|m| m.session_id.as_ref())
        {
            ensure!(session.len() <= 256, "session id exceeds 256 bytes");
            let identity = (route_id.as_str().to_owned(), session.clone(), agent);
            let mut sessions = self.sessions.lock().unwrap_or_else(|p| p.into_inner());
            let now = Instant::now();
            if !sessions.contains_key(&identity) && sessions.len() >= 4096 {
                sessions.retain(|_, seen| now.duration_since(*seen) < SESSION_TTL);
            }
            if !sessions.contains_key(&identity) && sessions.len() >= 4096 {
                return Err(
                    crate::server::Error::new(503, "session identity capacity exceeded").into(),
                );
            }
            // The SDK can retain session state even when decide fails or is cancelled.
            sessions.insert(identity.clone(), now);
            Some((identity, now))
        } else {
            None
        };
        let mut original_ir = request.llm_request.clone();
        original_ir.model = None;
        // The server's deadline includes body receipt and the classifier call.
        let mut outcome = route
            .decide(request)
            .await
            .map_err(|_| crate::server::Error::new(500, "SDK routing failed"))?;
        let inflight = self
            .metrics
            .as_ref()
            .map(|metrics| metrics.track(route, &mut outcome))
            .transpose()?;
        if is_session_final && let Some((identity, seen)) = admission {
            let mut sessions = self.sessions.lock().unwrap_or_else(|p| p.into_inner());
            // A newer request for this identity must retain its admission slot.
            if sessions.get(&identity) == Some(&seen) {
                sessions.remove(&identity);
            }
        }
        ensure!(outcome.response.is_none(), "routing-only contract violated");
        let model = route.provider_model(outcome.selected_model_id()?).clone();
        let mut selected_ir = outcome.request.llm_request;
        selected_ir.model = None;
        ensure!(
            selected_ir == original_ir,
            "router unexpectedly rewrote request semantics"
        );
        let output = replace_model(body, model.as_str())?;
        let elapsed_us = started.elapsed().as_micros() as u64;
        tracing::info!(
            %model,
            elapsed_us,
            "routing decision"
        );
        Ok((output, model.as_str().to_owned(), inflight))
    }
}

fn replace_model(body: &[u8], model: &str) -> Result<Vec<u8>, serde_json::Error> {
    // Preserve original values, including precise numbers and provider-specific fields.
    let mut fields: BTreeMap<String, &RawValue> = serde_json::from_slice(body)?;
    let model = to_raw_value(model)?;
    fields.insert("model".to_owned(), &model);
    serde_json::to_vec(&fields)
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json::{Value, json};

    fn router() -> Router {
        Router::new(Runner::from_toml(include_str!("../config/routes.toml")).unwrap())
    }
    fn neutral() -> Value {
        json!({"model":"auto", "messages":[{"role":"user", "content":"Say hello"}], "stream":true, "temperature":0.2, "custom_vendor_field":{"preserve":true}, "nvext":{"custom":"keep"}})
    }
    fn session(id: &str) -> http::HeaderMap {
        let mut h = http::HeaderMap::new();
        h.insert("x-switchyard-session-id", id.parse().unwrap());
        h
    }

    #[tokio::test]
    async fn session_capacity_recovers_after_idle_expiry() {
        let r = router();
        let quiet = serde_json::to_vec(&neutral()).unwrap();
        {
            let mut sessions = r.sessions.lock().unwrap();
            for i in 0..4096 {
                sessions.insert(("auto".into(), i.to_string(), None), Instant::now());
            }
        }
        let error = r.decide(&quiet, &session("new")).await.unwrap_err();
        assert_eq!(
            error
                .downcast_ref::<crate::server::Error>()
                .unwrap()
                .status_code,
            503
        );
        let mut final_headers = session("0");
        final_headers.insert("x-switchyard-session-final", "true".parse().unwrap());
        r.decide(&quiet, &final_headers).await.unwrap();
        assert_eq!(r.sessions.lock().unwrap().len(), 4095);
        r.decide(&quiet, &session("new")).await.unwrap();
        assert_eq!(r.sessions.lock().unwrap().len(), 4096);
        {
            let mut sessions = r.sessions.lock().unwrap();
            for seen in sessions.values_mut() {
                *seen = Instant::now() - SESSION_TTL;
            }
        }
        r.decide(&quiet, &session("0")).await.unwrap();
        r.decide(&quiet, &session("new")).await.unwrap();
        let sessions = r.sessions.lock().unwrap();
        assert_eq!(sessions.len(), 2);
        assert!(sessions.contains_key(&("auto".into(), "0".into(), None)));
        assert!(sessions.contains_key(&("auto".into(), "new".into(), None)));
    }

    #[tokio::test]
    async fn preserves_original_values_including_precise_numbers_and_nested_fields() {
        let input = br#"{
            "mo\u0064el":"auto",
            "messages":[ {"role":"user", "content":"Hello"} ],
            "large_integer":18446744073709551617,
            "precise_decimal":0.12345678901234567890123456789,
            "custom\u005ffield":{ "nested": [18446744073709551617, {"text":"\u0061"}] }
        }"#;
        let (output, model, _) = router()
            .decide(input, &http::HeaderMap::new())
            .await
            .unwrap();
        assert_eq!(model, "Qwen/Qwen3-0.6B");
        let original: HashMap<String, &serde_json::value::RawValue> =
            serde_json::from_slice(input).unwrap();
        let rewritten: HashMap<String, &serde_json::value::RawValue> =
            serde_json::from_slice(&output).unwrap();
        assert_eq!(original.len(), rewritten.len());
        for (key, value) in original {
            if key == "model" {
                assert_eq!(rewritten[&key].get(), r#""Qwen/Qwen3-0.6B""#);
            } else {
                assert_eq!(rewritten[&key].get(), value.get(), "changed {key}");
            }
        }
        let classifier =
            Router::new(Runner::from_toml(include_str!("../config/task-classifier.toml")).unwrap());
        let direct = replace_model(input, "Qwen/Qwen3-1.7B").unwrap();
        let (output, model, _) = classifier
            .decide(&direct, &http::HeaderMap::new())
            .await
            .unwrap();
        assert_eq!(model, "Qwen/Qwen3-1.7B");
        assert_eq!(output, direct);
    }
}