Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
10.7 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Embedded Transformers and checkpoint implementation of the complete forward pass.

use std::env;
use std::mem::size_of;
use std::path::PathBuf;

use pyo3::ffi::c_str;
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyModule};

use crate::error::python_error;
use crate::{PrefillForward, PrefillRouterError, Result};

/// Configuration for embedded Transformers prefill inference.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct TransformersForwardConfig {
    /// Tensor-only router checkpoint path.
    pub checkpoint: PathBuf,
    /// Torch device override. Auto-detected when omitted.
    pub device: Option<String>,
    /// Optional Hugging Face cache directory.
    pub cache_dir: Option<PathBuf>,
    /// Maximum tokenized prompt length.
    pub max_length: usize,
    /// Maximum prompts per encoder forward pass.
    pub batch_size: usize,
}

impl TransformersForwardConfig {
    /// Creates a configuration using automatic device and cache selection.
    pub fn new(checkpoint: impl Into<PathBuf>) -> Self {
        Self {
            checkpoint: checkpoint.into(),
            device: None,
            cache_dir: None,
            max_length: 2_048,
            batch_size: 32,
        }
    }
}

/// Embedded Python implementation of [`PrefillForward`].
pub struct TransformersForward {
    inner: Py<PyAny>,
    model: String,
    output_count: usize,
    max_length: usize,
    batch_size: usize,
    loaded: bool,
}

impl TransformersForward {
    /// Loads checkpoint metadata and defers loading the encoder until first use.
    pub fn new(config: TransformersForwardConfig) -> Result<Self> {
        if config.max_length == 0 || config.batch_size == 0 {
            return Err(PrefillRouterError::InvalidConfiguration(
                "max_length and batch_size must be greater than zero".to_string(),
            ));
        }
        Python::attach(|py| {
            add_venv_site_packages(py)
                .map_err(|error| python_error("virtual environment setup", error))?;
            let module = PyModule::from_code(
                py,
                c_str!(include_str!("../python/transformers_forward.py")),
                c"transformers_forward.py",
                c"_switchyard_prefill_transformers",
            )
            .map_err(|error| python_error("module initialization", error))?;
            let kwargs = PyDict::new(py);
            kwargs
                .set_item("device", config.device)
                .and_then(|()| {
                    kwargs.set_item(
                        "cache_dir",
                        config.cache_dir.map(|path| path.into_os_string()),
                    )
                })
                .map_err(|error| python_error("configuration", error))?;
            let checkpoint = config.checkpoint.into_os_string();
            let inner = module
                .getattr("TransformersForward")
                .and_then(|class| class.call((checkpoint,), Some(&kwargs)))
                .map_err(|error| python_error("construction", error))?;
            let (model, output_count) = inner
                .call_method0("metadata")
                .and_then(|value| value.extract())
                .map_err(|error| python_error("checkpoint metadata", error))?;
            Ok(Self {
                inner: inner.unbind(),
                model,
                output_count,
                max_length: config.max_length,
                batch_size: config.batch_size,
                loaded: false,
            })
        })
    }
}

impl PrefillForward for TransformersForward {
    fn output_count(&self) -> usize {
        self.output_count
    }

    fn forward(&mut self, prompts: &[String]) -> Result<Vec<Vec<f32>>> {
        if prompts.is_empty() || prompts.iter().any(String::is_empty) {
            return Err(PrefillRouterError::InvalidRequest(
                "prompts must be non-empty".to_string(),
            ));
        }
        Python::attach(|py| {
            let bytes = self
                .inner
                .bind(py)
                .call_method1("forward", (prompts, self.batch_size, self.max_length))
                .and_then(|value| value.extract::<Vec<u8>>())
                .map_err(|error| python_error("forward", error))?;
            if !self.loaded {
                tracing::info!(model = %self.model, "prefill model loaded");
                self.loaded = true;
            }
            decode_probabilities(&bytes, prompts.len(), self.output_count)
        })
    }

    fn unload(&mut self) -> Result<()> {
        Python::attach(|py| {
            self.inner
                .bind(py)
                .call_method0("unload")
                .map_err(|error| python_error("unload", error))?;
            self.loaded = false;
            Ok(())
        })
    }
}

// Embedded Python does not discover an activated virtual environment because
// the host executable is Rust. Add its site-packages before importing dependencies.
fn add_venv_site_packages(py: Python<'_>) -> PyResult<()> {
    let Some(venv) = env::var_os("VIRTUAL_ENV") else {
        return Ok(());
    };
    let site_packages = if cfg!(windows) {
        PathBuf::from(venv).join("Lib").join("site-packages")
    } else {
        let version = PyModule::import(py, "sys")?.getattr("version_info")?;
        let major = version.getattr("major")?.extract::<u8>()?;
        let minor = version.getattr("minor")?.extract::<u8>()?;
        PathBuf::from(venv)
            .join("lib")
            .join(format!("python{major}.{minor}"))
            .join("site-packages")
    };
    PyModule::import(py, "site")?.call_method1("addsitedir", (site_packages,))?;
    Ok(())
}

fn decode_probabilities(
    bytes: &[u8],
    prompt_count: usize,
    output_count: usize,
) -> Result<Vec<Vec<f32>>> {
    let expected = prompt_count
        .checked_mul(output_count)
        .and_then(|values| values.checked_mul(size_of::<f32>()))
        .ok_or_else(|| PrefillRouterError::InvalidResult("output is too large".to_string()))?;
    if bytes.len() != expected {
        return Err(PrefillRouterError::InvalidResult(format!(
            "forward returned {} bytes, expected {expected}",
            bytes.len()
        )));
    }
    let values = bytes
        .as_chunks::<{ size_of::<f32>() }>()
        .0
        .iter()
        .map(|bytes| {
            let value = f32::from_ne_bytes(*bytes);
            if value.is_finite() && (0.0..=1.0).contains(&value) {
                Ok(value)
            } else {
                Err(PrefillRouterError::InvalidResult(
                    "forward returned an invalid probability".to_string(),
                ))
            }
        })
        .collect::<Result<Vec<_>>>()?;
    Ok(values
        .chunks_exact(output_count)
        .map(<[f32]>::to_vec)
        .collect())
}

#[cfg(test)]
fn decode_expected_probabilities(
    bytes: &[u8],
    prompt_count: usize,
    output_count: usize,
) -> Result<Vec<Vec<f32>>> {
    let expected = prompt_count
        .checked_mul(size_of::<f32>())
        .and_then(|row_bytes| row_bytes.checked_mul(output_count))
        .ok_or_else(|| PrefillRouterError::InvalidResult("expected output is too large".into()))?;
    if bytes.len() != expected {
        return Err(PrefillRouterError::InvalidResult(
            "expected probabilities have an invalid shape".into(),
        ));
    }
    Ok(bytes
        .as_chunks::<{ size_of::<f32>() }>()
        .0
        .iter()
        .map(|bytes| f32::from_ne_bytes(*bytes))
        .collect::<Vec<_>>()
        .chunks_exact(output_count)
        .map(<[f32]>::to_vec)
        .collect())
}

#[cfg(test)]
mod tests {
    use std::env;
    use std::path::Path;

    use super::*;

    #[test]
    #[ignore = "requires PREFILL_ROUTER_HANDOFF_DIR"]
    fn matches_handoff_from_raw_prompts_end_to_end() -> Result<()> {
        let directory = env::var_os("PREFILL_ROUTER_HANDOFF_DIR").ok_or_else(|| {
            PrefillRouterError::InvalidConfiguration(
                "PREFILL_ROUTER_HANDOFF_DIR must name the handoff directory".to_string(),
            )
        })?;
        let directory = Path::new(&directory);
        let config = TransformersForwardConfig::new(
            directory.join("selected_dual_oof_router_checkpoint.pt"),
        );
        let mut forward = TransformersForward::new(config)?;
        let test_data = directory.join("fold_0_test_data.npz").into_os_string();
        let (prompts, expected) = Python::attach(|py| {
            let data = PyModule::import(py, "numpy")
                .and_then(|numpy| numpy.call_method1("load", (test_data,)))
                .map_err(|error| python_error("handoff data loading", error))?;
            let prompts = data
                .get_item("prompt")
                .and_then(|value| value.call_method0("tolist"))
                .and_then(|value| value.extract::<Vec<String>>())
                .map_err(|error| python_error("handoff prompt decoding", error))?;
            let probabilities = data
                .get_item("expected_probabilities")
                .map_err(|error| python_error("handoff probability decoding", error))?;
            let (_, output_count) = probabilities
                .getattr("shape")
                .and_then(|shape| shape.extract::<(usize, usize)>())
                .map_err(|error| python_error("handoff probability decoding", error))?;
            let bytes = probabilities
                .call_method0("tobytes")
                .and_then(|value| value.extract::<Vec<u8>>())
                .map_err(|error| python_error("handoff probability decoding", error))?;
            let expected = decode_expected_probabilities(&bytes, prompts.len(), output_count)?;
            Ok((prompts, expected))
        })?;
        let actual = forward.forward(&prompts)?;
        assert_eq!(actual.len(), expected.len());
        // BF16 kernels can perturb confidences; every resulting route must remain exact.
        for (actual, expected) in actual.iter().zip(&expected) {
            let actual_decision = actual
                .iter()
                .enumerate()
                .max_by(|(_, left), (_, right)| left.total_cmp(right))
                .map(|(index, _)| index);
            let expected_decision = expected
                .iter()
                .enumerate()
                .max_by(|(_, left), (_, right)| left.total_cmp(right))
                .map(|(index, _)| index);
            assert_eq!(actual_decision, expected_decision);
        }
        let max_delta = actual
            .iter()
            .flatten()
            .zip(expected.iter().flatten())
            .map(|(actual, expected)| (actual - expected).abs())
            .fold(0.0_f32, f32::max);
        assert!(max_delta <= 0.04, "probability max delta: {max_delta}");
        Ok(())
    }
}