Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
3.8 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

use std::{collections::HashMap, sync::Arc};
use futures::StreamExt;
use switchyard_libsy::{
    Algorithm, LibsyError, PickerMode, RuntimeModels, StageRouter, StageRouterConfig, State, Step,
};
use switchyard_protocol::{Category, LlmResponse, Request, Response, completion_text, text_request};
use switchyard_translation::{WireFormat, decode_aggregated_response, encode_request};

#[tokio::main(flavor = "current_thread")]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    // Start with Efficient when tool history gives no reason to change models.
    let router = Arc::new(StageRouter::new(StageRouterConfig::new(
        PickerMode::EfficientFirst, 0.5,
    ))?);
    let efficient = "z-ai/glm-5.2";
    let capable = "anthropic/claude-opus-4.8";
    let client = OpenRouterClient {
        http: reqwest::Client::builder()
            .timeout(std::time::Duration::from_secs(60)).build()?,
        api_key: std::env::var("OPENROUTER_API_KEY")?,
    };
    // Assign model roles; Any lists the full pool.
    let models = Arc::new(RuntimeModels::new(HashMap::from([
        (Category::Efficient, vec![efficient.into()]),
        (Category::Capable, vec![capable.into()]),
        (Category::Any, vec![efficient.into(), capable.into()]),
    ])));
    let request = Request {
        llm_request: text_request(None, "Run the tests."),
        ..Default::default()
    };

    let mut steps = router.run_stream(request, models, State::default());
    while let Some(step) = steps.next().await {
        match step? {
            Step::CallModel(call) => {
                // Give libsy the future so pending routing calls can be canceled.
                let request = call.request.clone();
                call.respond(client.call(request)).await?;
            }
            Step::CallDecision(_) => {
                // This Stage preset makes no decision calls; other presets may need a client.
                return Err("A decision client is required for this algorithm.".into());
            }
            Step::Done(outcome) => {
                println!("Selected: {}", outcome.selected_model_id()?);
                // Stage has selected the model. Your application now makes the final call.
                let response = client.call(outcome.request).await?;
                if let LlmResponse::Agg(answer) = response.llm_response {
                    println!("Answer: {}", completion_text(&answer));
                }
            }
        }
    }
    Ok(())
}

struct OpenRouterClient {
    http: reqwest::Client,
    api_key: String,
}

impl OpenRouterClient {
    async fn call(&self, request: Request) -> switchyard_libsy::Result<Response> {
        // Convert libsy's request to the provider's wire format.
        let body = encode_request(&request.llm_request, WireFormat::OpenAiChat)
            .map_err(|error| LibsyError::external("encoding OpenRouter request", error))?;
        let response = self.http
            .post("https://openrouter.ai/api/v1/chat/completions")
            .bearer_auth(&self.api_key)
            .json(&body)
            .send().await
            .and_then(reqwest::Response::error_for_status)
            .map_err(|error| LibsyError::external("OpenRouter request", error))?;
        let upstream_headers = response.headers().clone();
        let body = response.json::<serde_json::Value>().await
            .map_err(|error| LibsyError::external("reading OpenRouter response", error))?;
        let answer = decode_aggregated_response(&body, WireFormat::OpenAiChat)
            .map_err(|error| LibsyError::external("decoding OpenRouter response", error))?;
        Ok(Response {
            llm_response: LlmResponse::Agg(answer),
            metadata: request.metadata,
            upstream_headers,
        })
    }
}