Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
25.6 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::{
    pin::{Pin, pin},
    sync::Arc,
    time::Duration,
};

use anyhow::{Context, ensure};
use futures_util::{Stream, StreamExt};
use tokio::{
    sync::{OwnedSemaphorePermit, Semaphore, mpsc},
    task::JoinSet,
    time::{Instant, timeout, timeout_at},
};
use tonic::{Request, Response, Status, Streaming};

use crate::{
    preprocessor::{MAX_BODY, headers},
    proto::envoy::{
        config::core::v3::{HeaderValue, HeaderValueOption},
        service::ext_proc::v3::{
            self as ext,
            external_processor_server::{ExternalProcessor, ExternalProcessorServer},
            processing_request::Request as Input,
            processing_response::Response as Output,
        },
    },
    router::{MODEL_HEADER, Router},
};

const REQUEST_TIMEOUT: Duration = Duration::from_secs(5);
const RESPONSE_TIMEOUT: Duration = Duration::from_secs(120);
const CHUNK_SIZE: usize = 64 * 1024;
const MAX_UPLOADS: usize = 8;

#[derive(Debug, thiserror::Error)]
#[error("{message}")]
pub struct Error {
    pub status_code: u16,
    pub message: String,
}

impl Error {
    pub fn new(status_code: u16, message: impl Into<String>) -> Self {
        Self {
            status_code,
            message: message.into(),
        }
    }
}

#[derive(Clone)]
pub struct Server {
    router: Arc<Router>,
    requests: Arc<Semaphore>,
    streams: Arc<Semaphore>,
    direct_streams: Arc<Semaphore>,
}

struct Prepared {
    body: Vec<u8>,
    headers: ext::HeaderMutation,
    has_trailers: bool,
    inflight: Option<crate::metrics::InFlight>,
    stream_permit: OwnedSemaphorePermit,
}

impl Server {
    pub fn new(router: Arc<Router>, max_active_streams: usize) -> Self {
        Self {
            router,
            requests: Arc::new(Semaphore::new(MAX_UPLOADS)),
            streams: Arc::new(Semaphore::new(max_active_streams)),
            direct_streams: Arc::new(Semaphore::new(max_active_streams)),
        }
    }

    pub fn into_service(self) -> ExternalProcessorServer<Self> {
        ExternalProcessorServer::new(self)
    }

    async fn prepare(
        &self,
        input: &mut Streaming<ext::ProcessingRequest>,
        request_permit: OwnedSemaphorePermit,
    ) -> anyhow::Result<Prepared> {
        let first = input.message().await?.context("missing request headers")?;
        let Some(Input::RequestHeaders(first)) = first.request else {
            anyhow::bail!("request headers must arrive first");
        };
        ensure!(
            !first.end_of_stream,
            "chat completion requires a request body"
        );
        let values = first
            .headers
            .unwrap_or_default()
            .headers
            .into_iter()
            .map(|h| {
                let value = if h.raw_value.is_empty() {
                    h.value
                } else {
                    String::from_utf8(h.raw_value)?
                };
                Ok((h.key, value))
            })
            .collect::<anyhow::Result<Vec<_>>>()?;
        let (headers, mut remove_headers) = headers(&values)?;
        let mut body = Vec::new();
        let mut has_trailers = false;
        loop {
            let next = input.message().await?.context("incomplete request body")?;
            match next.request {
                Some(Input::RequestBody(chunk)) => {
                    ensure!(
                        !has_trailers || (chunk.body.is_empty() && chunk.end_of_stream),
                        "expected body end after request trailers"
                    );
                    if chunk.body.len() > MAX_BODY - body.len() {
                        return Err(Error::new(413, "request body exceeds 2 MiB").into());
                    }
                    body.extend_from_slice(&chunk.body);
                    if chunk.end_of_stream {
                        break;
                    }
                }
                // Stock agentgateway 1.0 sends an empty body EOS after trailers.
                Some(Input::RequestTrailers(_)) if !has_trailers => has_trailers = true,
                _ => anyhow::bail!("expected request body"),
            }
        }
        // Loopback judge calls need headroom while the parent holds its stream slot.
        let streams = if self.router.is_passthrough(&body) {
            &self.direct_streams
        } else {
            &self.streams
        };
        let stream_permit = streams
            .clone()
            .try_acquire_owned()
            .map_err(|_| Error::new(503, "processor stream limit reached"))?;
        drop(request_permit);
        let (body, model, inflight) = self.router.decide(&body, &headers).await?;
        if body.len() > MAX_BODY {
            return Err(Error::new(413, "rewritten request body exceeds 2 MiB").into());
        }
        remove_headers.push("content-length".into());
        Ok(Prepared {
            body,
            headers: ext::HeaderMutation {
                set_headers: vec![set_header(MODEL_HEADER, &model)],
                remove_headers,
            },
            has_trailers,
            inflight,
            stream_permit,
        })
    }
}

#[tonic::async_trait]
impl ExternalProcessor for Server {
    type ProcessStream =
        Pin<Box<dyn Stream<Item = Result<ext::ProcessingResponse, Status>> + Send>>;

    async fn process(
        &self,
        request: Request<Streaming<ext::ProcessingRequest>>,
    ) -> Result<Response<Self::ProcessStream>, Status> {
        let request_permit = self.requests.clone().try_acquire_owned();
        let server = self.clone();
        let deadline = Instant::now() + REQUEST_TIMEOUT;
        let mut input = request.into_inner();
        let output = async_stream::stream! {
            let Ok(request_permit) = request_permit else {
                yield Ok(reject(Error::new(503, "request upload limit reached")));
                return;
            };
            let result = timeout_at(deadline, server.prepare(&mut input, request_permit)).await;
            let prepared = match result {
                Ok(Ok(prepared)) => prepared,
                Ok(Err(error)) => {
                    tracing::warn!(%error, "preprocessing rejected request");
                    yield Ok(reject(error.downcast::<Error>()
                        .unwrap_or_else(|e| Error::new(400, e.to_string()))));
                    return;
                }
                Err(_) => {
                    yield Ok(reject(Error::new(504, "preprocessing deadline exceeded")));
                    return;
                }
            };
            // Both guards live through response completion, cancellation and backpressure.
            let _stream_permit = prepared.stream_permit;
            let _inflight = prepared.inflight;
            let mut response_deadline = Instant::now() + RESPONSE_TIMEOUT;
            // v1.0 selects the route when this header reply arrives, before reading the body.
            yield Ok(reply(Output::RequestHeaders(ext::HeadersResponse {
                response: Some(ext::CommonResponse {
                    header_mutation: Some(prepared.headers),
                    ..Default::default()
                }),
            })));
            if prepared.has_trailers {
                yield Ok(reply(Output::RequestTrailers(ext::TrailersResponse::default())));
            }
            for (i, chunk) in prepared.body.chunks(CHUNK_SIZE).enumerate() {
                if Instant::now() >= response_deadline {
                    yield Err(Status::deadline_exceeded("request output timed out"));
                    return;
                }
                yield Ok(reply(Output::RequestBody(body_reply(
                    chunk.to_vec(), (i + 1) * CHUNK_SIZE >= prepared.body.len(),
                ))));
            }
            response_deadline = Instant::now() + RESPONSE_TIMEOUT;
            let mut has_headers = false;
            let mut has_trailers = false;
            loop {
                let next = match timeout_at(response_deadline, input.message()).await {
                    Ok(Ok(Some(next))) => next,
                    Ok(Ok(None)) => {
                        yield Err(Status::failed_precondition("incomplete response stream"));
                        return;
                    }
                    Ok(Err(error)) => { yield Err(error); return; }
                    Err(_) => {
                        yield Err(Status::deadline_exceeded("response processing timed out"));
                        return;
                    }
                };
                response_deadline = Instant::now() + RESPONSE_TIMEOUT;
                let (output, is_done) = match next.request {
                    Some(Input::ResponseHeaders(h)) if !has_headers => {
                        has_headers = true;
                        (Output::ResponseHeaders(ext::HeadersResponse {
                            response: Some(ext::CommonResponse::default()),
                        }), h.end_of_stream)
                    }
                    Some(Input::ResponseBody(b)) if has_headers
                        && (!has_trailers || (b.body.is_empty() && b.end_of_stream)) => {
                        let is_done = b.end_of_stream;
                        (Output::ResponseBody(body_reply(b.body, is_done)), is_done)
                    }
                    Some(Input::ResponseTrailers(_)) if has_headers && !has_trailers => {
                        has_trailers = true;
                        // ACK trailers, then consume agentgateway's synthetic body EOS.
                        (Output::ResponseTrailers(ext::TrailersResponse::default()), false)
                    }
                    _ => {
                        yield Err(Status::invalid_argument("unexpected response processing phase"));
                        return;
                    }
                };
                yield Ok(reply(output));
                if is_done { return; }
            }
        };
        Ok(Response::new(Box::pin(with_output_timeout(output))))
    }
}

fn with_output_timeout(
    output: impl Stream<Item = Result<ext::ProcessingResponse, Status>> + Send + 'static,
) -> impl Stream<Item = Result<ext::ProcessingResponse, Status>> + Send {
    let (tx, mut rx) = mpsc::channel(1);
    let mut tasks = JoinSet::new();
    // Poll independently of tonic so a stalled reader cannot retain admission permits.
    tasks.spawn(async move {
        let mut output = pin!(output);
        while let Some(response) = output.next().await {
            if timeout(RESPONSE_TIMEOUT, tx.send(response))
                .await
                .map_err(|_| Status::deadline_exceeded("processor output stalled"))?
                .is_err()
            {
                break;
            }
        }
        Ok::<_, Status>(())
    });
    async_stream::stream! {
        // Owning the JoinSet also cancels the producer when this stream is dropped.
        while let Some(response) = rx.recv().await {
            yield response;
        }
        match tasks.join_next().await {
            Some(Ok(Err(error))) => yield Err(error),
            Some(Err(_)) => yield Err(Status::internal("processor task failed")),
            _ => {}
        }
    }
}

fn reply(response: Output) -> ext::ProcessingResponse {
    ext::ProcessingResponse {
        response: Some(response),
        ..Default::default()
    }
}

fn body_reply(body: Vec<u8>, is_end_of_stream: bool) -> ext::BodyResponse {
    ext::BodyResponse {
        response: Some(ext::CommonResponse {
            body_mutation: Some(ext::BodyMutation {
                mutation: Some(ext::body_mutation::Mutation::StreamedResponse(
                    ext::StreamedBodyResponse {
                        body,
                        end_of_stream: is_end_of_stream,
                    },
                )),
            }),
            ..Default::default()
        }),
    }
}

fn set_header(key: &str, value: &str) -> HeaderValueOption {
    HeaderValueOption {
        header: Some(HeaderValue {
            key: key.into(),
            value: String::new(),
            raw_value: value.as_bytes().to_vec(),
        }),
        append_action: 2,
        keep_empty_value: false,
        ..Default::default()
    }
}

fn reject(error: Error) -> ext::ProcessingResponse {
    reply(Output::ImmediateResponse(ext::ImmediateResponse {
        status: Some(crate::proto::envoy::r#type::v3::HttpStatus {
            code: i32::from(error.status_code),
        }),
        headers: Some(ext::HeaderMutation {
            set_headers: vec![set_header("content-type", "application/json")],
            remove_headers: vec![],
        }),
        body: serde_json::to_vec(&serde_json::json!({"error": {
            "message": error.message, "type": "switchyard_preproc_error"
        }}))
        .expect("JSON error object serializes"),
        details: "switchyard_preproc".into(),
        ..Default::default()
    }))
}

#[cfg(test)]
mod tests {
    use super::*;
    use tokio_stream::wrappers::{ReceiverStream, TcpListenerStream};

    type Client =
        ext::external_processor_client::ExternalProcessorClient<tonic::transport::Channel>;

    async fn start(
        max_active_streams: usize,
        config: &str,
    ) -> (Client, Arc<Semaphore>, Arc<Router>) {
        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
        let address = listener.local_addr().unwrap();
        let runner = switchyard_runner::Runner::from_toml(config);
        let mut router = Router::new(runner.unwrap());
        router.metrics = Some(crate::metrics::Metrics::empty_for_test());
        let router = Arc::new(router);
        let server = Server::new(router.clone(), max_active_streams);
        let requests = server.requests.clone();
        tokio::spawn(async move {
            tonic::transport::Server::builder()
                .add_service(server.into_service())
                .serve_with_incoming(TcpListenerStream::new(listener))
                .await
                .unwrap();
        });
        (
            Client::new(
                tonic::transport::Endpoint::from_shared(format!("http://{address}"))
                    .unwrap()
                    .connect()
                    .await
                    .unwrap(),
            ),
            requests,
            router,
        )
    }

    fn input(request: Input) -> ext::ProcessingRequest {
        ext::ProcessingRequest {
            request: Some(request),
            ..Default::default()
        }
    }

    fn body(bytes: &[u8], is_end_of_stream: bool) -> ext::HttpBody {
        ext::HttpBody {
            body: bytes.to_vec(),
            end_of_stream: is_end_of_stream,
        }
    }

    fn headers() -> Input {
        let headers = [
            (":method", "POST"),
            (":path", "/v1/chat/completions"),
            ("content-type", "application/json"),
            (MODEL_HEADER, "forged"),
        ]
        .into_iter()
        .map(|(k, v)| set_header(k, v).header.unwrap())
        .collect();
        Input::RequestHeaders(ext::HttpHeaders {
            headers: Some(crate::proto::envoy::config::core::v3::HeaderMap { headers }),
            end_of_stream: false,
            ..Default::default()
        })
    }

    async fn send(tx: &mpsc::Sender<ext::ProcessingRequest>, request: Input) {
        tx.send(input(request)).await.unwrap();
    }

    async fn next(rx: &mut Streaming<ext::ProcessingResponse>) -> Output {
        rx.message().await.unwrap().unwrap().response.unwrap()
    }

    async fn process_body(client: &mut Client, json: &[u8]) -> Streaming<ext::ProcessingResponse> {
        client
            .process(tokio_stream::iter([
                input(headers()),
                input(Input::RequestBody(body(json, true))),
            ]))
            .await
            .unwrap()
            .into_inner()
    }

    fn streamed(reply: ext::BodyResponse) -> ext::StreamedBodyResponse {
        let Some(ext::body_mutation::Mutation::StreamedResponse(body)) =
            reply.response.unwrap().body_mutation.unwrap().mutation
        else {
            panic!()
        };
        body
    }

    #[tokio::test]
    async fn routes_upload_and_passes_response_through_trailers_to_eos() {
        let (mut client, requests, router) = start(16, include_str!("../config/routes.toml")).await;
        let (tx, receive) = mpsc::channel(8);
        send(&tx, headers()).await;
        let receive = ReceiverStream::new(receive);
        let mut rx = client.process(receive).await.unwrap().into_inner();
        let json = br#"{"model":"auto","messages":[{"role":"user","content":"hello"}]}"#;
        for chunk in json.chunks(40) {
            send(&tx, Input::RequestBody(body(chunk, false))).await;
        }
        assert!(
            tokio::time::timeout(Duration::from_millis(30), rx.message())
                .await
                .is_err()
        );
        send(&tx, Input::RequestTrailers(ext::HttpTrailers::default())).await;
        send(&tx, Input::RequestBody(body(b"", true))).await;
        let Output::RequestHeaders(h) = next(&mut rx).await else {
            panic!()
        };
        let h = h.response.unwrap().header_mutation.unwrap();
        let selected = h.set_headers[0].header.as_ref().unwrap();
        assert_eq!(selected.raw_value, b"Qwen/Qwen3-0.6B");
        assert!(h.remove_headers.contains(&"content-length".to_owned()));
        assert!(matches!(next(&mut rx).await, Output::RequestTrailers(_)));
        let Output::RequestBody(b) = next(&mut rx).await else {
            panic!()
        };
        let b = streamed(b);
        assert!(b.end_of_stream);
        let value: serde_json::Value = serde_json::from_slice(&b.body).unwrap();
        assert_eq!(value["model"], "Qwen/Qwen3-0.6B");
        assert_eq!(requests.available_permits(), MAX_UPLOADS);
        assert_eq!(router.metrics.as_ref().unwrap().inflight_for_test(), 1);
        send(&tx, Input::ResponseHeaders(ext::HttpHeaders::default())).await;
        assert!(matches!(next(&mut rx).await, Output::ResponseHeaders(_)));
        for (bytes, is_done) in [
            (b"data: hello\n\n".as_slice(), false),
            (b"".as_slice(), true),
        ] {
            send(&tx, Input::ResponseBody(body(bytes, is_done))).await;
            let Output::ResponseBody(b) = next(&mut rx).await else {
                panic!()
            };
            let b = streamed(b);
            assert_eq!((b.body.as_slice(), b.end_of_stream), (bytes, is_done));
            if !is_done {
                send(&tx, Input::ResponseTrailers(ext::HttpTrailers::default())).await;
                assert!(matches!(next(&mut rx).await, Output::ResponseTrailers(_)));
            }
        }
        assert!(rx.message().await.unwrap().is_none());
        assert_eq!(router.metrics.as_ref().unwrap().inflight_for_test(), 0);

        let (tx, receive) = mpsc::channel(8);
        send(&tx, headers()).await;
        send(&tx, Input::RequestBody(body(json, true))).await;
        let mut rx = client
            .process(ReceiverStream::new(receive))
            .await
            .unwrap()
            .into_inner();
        assert!(matches!(next(&mut rx).await, Output::RequestHeaders(_)));
        assert!(matches!(next(&mut rx).await, Output::RequestBody(_)));
        assert_eq!(router.metrics.as_ref().unwrap().inflight_for_test(), 1);
        drop(rx);
        drop(tx);
        tokio::time::timeout(Duration::from_secs(1), async {
            while router.metrics.as_ref().unwrap().inflight_for_test() != 0 {
                tokio::time::sleep(Duration::from_millis(5)).await;
            }
        })
        .await
        .unwrap();
    }

    #[tokio::test]
    async fn response_timeout_tracks_inactivity() {
        async fn elapse(duration: Duration) {
            // Resume before socket I/O so idle network polls cannot auto-advance time.
            tokio::time::pause();
            tokio::time::advance(duration).await;
            tokio::time::resume();
        }

        let (mut client, _, router) = start(1, include_str!("../config/routes.toml")).await;
        let (tx, receive) = mpsc::channel(8);
        send(&tx, headers()).await;
        let mut rx = client
            .process(ReceiverStream::new(receive))
            .await
            .unwrap()
            .into_inner();
        send(
            &tx,
            Input::RequestBody(body(
                br#"{"model":"auto","messages":[{"role":"user","content":"hello"}]}"#,
                true,
            )),
        )
        .await;
        assert!(matches!(next(&mut rx).await, Output::RequestHeaders(_)));
        assert!(matches!(next(&mut rx).await, Output::RequestBody(_)));
        send(&tx, Input::ResponseHeaders(ext::HttpHeaders::default())).await;
        assert!(matches!(next(&mut rx).await, Output::ResponseHeaders(_)));

        let mut rejected = process_body(
            &mut client,
            br#"{"model":"auto","messages":[{"role":"user","content":"hello"}]}"#,
        )
        .await;
        let Output::ImmediateResponse(error) = next(&mut rejected).await else {
            panic!()
        };
        assert_eq!(error.status.unwrap().code, 503);

        for _ in 0..3 {
            elapse(RESPONSE_TIMEOUT / 2).await;
            send(&tx, Input::ResponseBody(body(b"data: hello\n\n", false))).await;
            let Output::ResponseBody(b) = next(&mut rx).await else {
                panic!()
            };
            assert_eq!(streamed(b).body, b"data: hello\n\n");
        }
        assert_eq!(router.metrics.as_ref().unwrap().inflight_for_test(), 1);
        elapse(RESPONSE_TIMEOUT + Duration::from_secs(1)).await;
        assert_eq!(
            rx.message().await.unwrap_err().code(),
            tonic::Code::DeadlineExceeded
        );
        assert_eq!(router.metrics.as_ref().unwrap().inflight_for_test(), 0);
    }

    #[tokio::test]
    async fn rejects_cumulative_body_limit_before_routing_headers() {
        let (mut client, _, _) = start(16, include_str!("../config/routes.toml")).await;
        let messages = tokio_stream::iter([
            input(headers()),
            input(Input::RequestBody(body(&vec![b' '; MAX_BODY], false))),
            input(Input::RequestBody(body(b" ", true))),
        ]);
        let mut rx = client.process(messages).await.unwrap().into_inner();
        let Output::ImmediateResponse(error) = next(&mut rx).await else {
            panic!()
        };
        assert_eq!(error.status.unwrap().code, 413);
        assert!(rx.message().await.unwrap().is_none());
    }

    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
    async fn loopback_judge_runs_when_task_slots_are_full() {
        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
        let address = listener.local_addr().unwrap();
        let config = include_str!("../config/task-classifier.toml")
            .replace(
                "http://inference-gateway.agentgateway-system.svc.cluster.local/v1",
                &format!("http://{address}/v1"),
            )
            .replace("fail_open = true", "fail_open = false");
        let (client, _, _) = start(8, &config).await;
        let nested_client = client.clone();
        let barrier = Arc::new(tokio::sync::Barrier::new(8));
        let app = axum::Router::new().route("/v1/chat/completions", axum::routing::post(move || {
            let mut client = nested_client.clone();
            let barrier = barrier.clone();
            async move {
                // Hold all parent slots before their judge calls re-enter PreProc.
                barrier.wait().await;
                let json = br#"{"model":"Qwen/Qwen3-1.7B","messages":[{"role":"user","content":"judge"}]}"#;
                let mut rx = process_body(&mut client, json).await;
                assert!(matches!(next(&mut rx).await, Output::RequestHeaders(_)));
                assert!(matches!(next(&mut rx).await, Output::RequestBody(_)));
                axum::Json(serde_json::json!({
                    "id":"judge", "object":"chat.completion", "created":0, "model":"Qwen/Qwen3-1.7B",
                    "choices":[{"index":0,"message":{"role":"assistant","content":
                        r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#
                    },"finish_reason":"stop"}]
                }))
            }
        }));
        let http_task = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
        let mut tasks = JoinSet::new();
        for _ in 0..8 {
            let mut client = client.clone();
            tasks.spawn(async move {
                let json =
                    br#"{"model":"auto","messages":[{"role":"user","content":"Sort [3,1,2]."}]}"#;
                let mut rx = process_body(&mut client, json).await;
                assert!(matches!(next(&mut rx).await, Output::RequestHeaders(_)));
                let Output::RequestBody(response) = next(&mut rx).await else {
                    panic!("expected routed body")
                };
                let body: serde_json::Value =
                    serde_json::from_slice(&streamed(response).body).unwrap();
                assert_eq!(body["model"], "Qwen/Qwen3-0.6B");
            });
        }
        while let Some(result) = tasks.join_next().await {
            result.unwrap();
        }
        http_task.abort();
    }
}