Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
3.7 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

mod metrics;
mod preprocessor;
mod request;
mod router;
mod server;

use envoy_types::pb as proto;

use std::{
    sync::{
        Arc,
        atomic::{AtomicBool, Ordering},
    },
    time::Duration,
};

use anyhow::{Context, ensure};
use axum::{Router as HttpRouter, extract::State, http::StatusCode, routing::get};
use tokio::sync::{Semaphore, watch};
use tonic::transport::Server;

use crate::{preprocessor::MAX_BODY, router::Router};

async fn readiness(State(ready): State<Arc<AtomicBool>>) -> StatusCode {
    if ready.load(Ordering::Relaxed) {
        StatusCode::OK
    } else {
        StatusCode::SERVICE_UNAVAILABLE
    }
}

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    tracing_subscriber::fmt()
        .with_env_filter(
            tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()),
        )
        .init();
    let max_active_streams = std::env::var("MAX_ACTIVE_STREAMS")
        .unwrap_or_else(|_| "16".into())
        .parse::<usize>()
        .context("MAX_ACTIVE_STREAMS must be a positive integer")?;
    ensure!(
        (1..=Semaphore::MAX_PERMITS).contains(&max_active_streams),
        "MAX_ACTIVE_STREAMS must be between 1 and {}",
        Semaphore::MAX_PERMITS
    );
    let router = Arc::new(Router::load(
        std::env::var("ROUTES_CONFIG").unwrap_or_else(|_| "config/routes.toml".into()),
    )?);
    let ready = Arc::new(AtomicBool::new(true));
    let (shutdown, receiver) = watch::channel(false);
    let http = HttpRouter::new()
        .route("/healthz", get(|| async { StatusCode::OK }))
        .route("/readyz", get(readiness))
        .with_state(ready.clone());
    let listener = tokio::net::TcpListener::bind(
        std::env::var("ADMIN_ADDR").unwrap_or_else(|_| "0.0.0.0:9003".into()),
    )
    .await?;
    let mut http_shutdown = receiver.clone();
    let admin_task = tokio::spawn(async move {
        axum::serve(listener, http)
            .with_graceful_shutdown(async move {
                let _ = http_shutdown.changed().await;
            })
            .await
    });
    let processor = server::Server::new(router, max_active_streams)
        .into_service()
        .max_decoding_message_size(MAX_BODY + 65536)
        .max_encoding_message_size(MAX_BODY + 65536);
    let addr = std::env::var("GRPC_ADDR")
        .unwrap_or_else(|_| "0.0.0.0:9002".into())
        .parse()?;
    tracing::info!(%addr, max_active_streams, "configured decision-only preprocessor listening");
    let grpc = Server::builder()
        .http2_keepalive_interval(Some(Duration::from_secs(30)))
        .add_service(processor)
        .serve_with_shutdown(addr, async move {
            let mut terminate =
                tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
                    .expect("install SIGTERM handler");
            tokio::select! { _ = tokio::signal::ctrl_c() => {}, _ = terminate.recv() => {} }
            ready.store(false, Ordering::Relaxed);
            let _ = shutdown.send(true);
        });
    grpc.await?;
    admin_task.await??;
    Ok(())
}