Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
mod metrics;
mod preprocessor;
mod request;
mod router;
mod server;
use envoy_types::pb as proto;
use std::{
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
time::Duration,
};
use anyhow::{Context, ensure};
use axum::{Router as HttpRouter, extract::State, http::StatusCode, routing::get};
use tokio::sync::{Semaphore, watch};
use tonic::transport::Server;
use crate::{preprocessor::MAX_BODY, router::Router};
async fn readiness(State(ready): State<Arc<AtomicBool>>) -> StatusCode {
if ready.load(Ordering::Relaxed) {
StatusCode::OK
} else {
StatusCode::SERVICE_UNAVAILABLE
}
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()),
)
.init();
let max_active_streams = std::env::var("MAX_ACTIVE_STREAMS")
.unwrap_or_else(|_| "16".into())
.parse::<usize>()
.context("MAX_ACTIVE_STREAMS must be a positive integer")?;
ensure!(
(1..=Semaphore::MAX_PERMITS).contains(&max_active_streams),
"MAX_ACTIVE_STREAMS must be between 1 and {}",
Semaphore::MAX_PERMITS
);
let router = Arc::new(Router::load(
std::env::var("ROUTES_CONFIG").unwrap_or_else(|_| "config/routes.toml".into()),
)?);
let ready = Arc::new(AtomicBool::new(true));
let (shutdown, receiver) = watch::channel(false);
let http = HttpRouter::new()
.route("/healthz", get(|| async { StatusCode::OK }))
.route("/readyz", get(readiness))
.with_state(ready.clone());
let listener = tokio::net::TcpListener::bind(
std::env::var("ADMIN_ADDR").unwrap_or_else(|_| "0.0.0.0:9003".into()),
)
.await?;
let mut http_shutdown = receiver.clone();
let admin_task = tokio::spawn(async move {
axum::serve(listener, http)
.with_graceful_shutdown(async move {
let _ = http_shutdown.changed().await;
})
.await
});
let processor = server::Server::new(router, max_active_streams)
.into_service()
.max_decoding_message_size(MAX_BODY + 65536)
.max_encoding_message_size(MAX_BODY + 65536);
let addr = std::env::var("GRPC_ADDR")
.unwrap_or_else(|_| "0.0.0.0:9002".into())
.parse()?;
tracing::info!(%addr, max_active_streams, "configured decision-only preprocessor listening");
let grpc = Server::builder()
.http2_keepalive_interval(Some(Duration::from_secs(30)))
.add_service(processor)
.serve_with_shutdown(addr, async move {
let mut terminate =
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
.expect("install SIGTERM handler");
tokio::select! { _ = tokio::signal::ctrl_c() => {}, _ = terminate.recv() => {} }
ready.store(false, Ordering::Relaxed);
let _ = shutdown.send(true);
});
grpc.await?;
admin_task.await??;
Ok(())
}