Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
9.7 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

use std::sync::{Arc, mpsc};

use serde_json::{Value, json};
use switchyard_llm_client::RunObservation;
use switchyard_protocol::{Metadata, Request, Usage, text_request};
use switchyard_runner::{AlgorithmFactory, Runner};
use wiremock::matchers::{body_partial_json, header, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};

#[tokio::test]
async fn openai_decisions_routes_from_toml() -> Result<(), Box<dyn std::error::Error>> {
    let server = MockServer::start().await;
    let endpoint = server.uri();
    let source = format!(
        r#"
schema_version = 1
[llm_clients.answers]
format = "openai_chat"
base_url = "{endpoint}/v1"
[targets]
strong = {{ id = "strong/model", llm_client = "answers" }}
weak = {{ id = "weak/model", llm_client = "answers" }}
[decision_clients.judge]
format = "openai_decisions"
endpoint = "{endpoint}/v1/decisions"
api_key_env = "SWITCHYARD_OPENAI_DECISIONS_TEST_KEY"
timeout_ms = 2000
[decision_targets.judge]
id = "gpt-6-luna"
decision_client = "judge"
[routes.classifier]
id = "switchyard/classifier"
type = "llm_classifier"
classifier_target = "judge"
strong_target = "strong"
weak_target = "weak"
fail_open = false
[routes.classifier.decision]
cutoff = 0.92
instructions = "Choose a Stage profile."
apply_cutoff_to = "capable_first"
choices = [
    {{ id = "capable_first", description = "Start capable" }},
    {{ id = "efficient_first", description = "Start efficient" }},
]
"#
    );
    // This unique variable is only read while constructing this client.
    unsafe {
        std::env::set_var("SWITCHYARD_OPENAI_DECISIONS_TEST_KEY", "test-key");
    }
    let runner = Runner::from_toml(&source);
    unsafe {
        std::env::remove_var("SWITCHYARD_OPENAI_DECISIONS_TEST_KEY");
    }
    let runner = runner?;
    let question = json!({
        "name": "route", "type": "choice",
        "instructions": "Choose a Stage profile.",
        "choices": [{"value": "capable_first", "description": "Start capable"},
            {"value": "efficient_first", "description": "Start efficient"}]
    });
    Mock::given(method("POST")).and(path("/v1/decisions"))
        .and(header("authorization", "Bearer test-key"))
        .and(body_partial_json(json!({"model": "gpt-6-luna", "questions": [question]})))
        .respond_with(ResponseTemplate::new(200).set_body_json(json!({
            "model": "gpt-6-luna", "answers": [{
                "type": "choice", "name": "route", "choice": "capable_first", "confidence": 0.99,
                "probabilities": [{"value": "capable_first", "probability": 0.91},
                    {"value": "efficient_first", "probability": 0.09}]
            }],
            "usage": {"input_tokens": 42, "input_tokens_details": {"cached_tokens": 10, "cache_write_tokens": 2},
                "output_tokens": 3, "output_tokens_details": {"reasoning_tokens": 1}, "total_tokens": 45}
        }))).expect(1).mount(&server).await;
    Mock::given(method("POST")).and(path("/v1/chat/completions"))
        .and(body_partial_json(json!({"model": "weak/model"})))
        .respond_with(ResponseTemplate::new(200).set_body_json(json!({
            "id": "answer-1", "object": "chat.completion", "model": "weak/model",
            "choices": [{"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"}]
        }))).expect(1).mount(&server).await;
    let (sender, observations) = mpsc::channel();
    let output = runner
        .route("switchyard/classifier")
        .unwrap()
        .execute(
            Request {
                llm_request: text_request(None, "Route this request"),
                metadata: Some(Metadata {
                    session_id: Some("session-1".into()),
                    ..Metadata::default()
                }),
                ..Request::default()
            },
            Some(Arc::new(move |event| {
                sender.send(event).unwrap();
            })),
        )
        .await?;
    assert_eq!(output.selected_model, "weak");
    assert_eq!(
        runner
            .route("switchyard/classifier")
            .unwrap()
            .provider_model(&output.selected_model),
        "weak/model"
    );
    let usage = observations.try_iter().find_map(|event| match event {
        RunObservation::DecisionCall(call) => call.usage,
        _ => None,
    });
    assert_eq!(
        usage,
        Some(Usage {
            input_tokens: Some(30),
            cache: Usage::cache_details(Some(10), Some(2)),
            output_tokens: Some(3),
            total_tokens: Some(45),
            reasoning_tokens: Some(1),
            ..Usage::default()
        })
    );
    let requests = server.received_requests().await.unwrap();
    let decision: Value = serde_json::from_slice(&requests[0].body)?;
    assert_eq!(
        decision["input"],
        serde_json::to_string(&text_request(None, "Route this request").messages)?
    );
    server.verify().await;
    Ok(())
}

fn direct_config(endpoint: &str) -> String {
    format!(
        r#"
schema_version = 1
[llm_clients.answers]
format = "openai_chat"
base_url = "{endpoint}/v1"
[targets]
fast = {{ id = "fast/model", llm_client = "answers" }}
balanced = {{ id = "balanced/model", llm_client = "answers" }}
premium = {{ id = "premium/model", llm_client = "answers" }}
[decision_clients.router]
format = "openai_decisions"
endpoint = "{endpoint}/v1/decisions"
api_key_env = "SWITCHYARD_DIRECT_DECISIONS_TEST_KEY"
timeout_ms = 2000
[decision_targets.router]
id = "router-model"
decision_client = "router"
[routes.direct]
id = "switchyard/direct"
type = "llm_classifier"
mode = "direct"
default_target = "fast"
[routes.direct.models]
judge = ["router"]
any = ["fast", "balanced", "premium"]
fast = ["fast"]
balanced = ["balanced"]
premium = ["premium"]
unused = []
"#
    )
}

#[tokio::test]
async fn direct_mode_serves_the_selected_group() -> Result<(), Box<dyn std::error::Error>> {
    let server = MockServer::start().await;
    // This unique variable is only read while constructing this client.
    unsafe {
        std::env::set_var("SWITCHYARD_DIRECT_DECISIONS_TEST_KEY", "test-key");
    }
    let runner = Runner::from_toml(&direct_config(&server.uri()));
    unsafe {
        std::env::remove_var("SWITCHYARD_DIRECT_DECISIONS_TEST_KEY");
    }
    let runner = runner?;
    // The choice routes even though another option has the higher probability.
    Mock::given(method("POST")).and(path("/v1/decisions"))
        .and(body_partial_json(json!({"model": "router-model", "questions": [{
            "name": "route", "type": "choice",
            "choices": [
                {"value": "balanced", "description": "Serve this request with the balanced models."},
                {"value": "fast", "description": "Serve this request with the fast models."},
                {"value": "premium", "description": "Serve this request with the premium models."}
            ]
        }]})))
        .respond_with(ResponseTemplate::new(200).set_body_json(json!({
            "model": "router-model", "answers": [{
                "type": "choice", "name": "route", "choice": "premium",
                "probabilities": [{"value": "balanced", "probability": 0},
                    {"value": "fast", "probability": 1}, {"value": "premium", "probability": 0}]
            }],
            "usage": {"input_tokens": 0, "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
                "output_tokens": 0, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 0}
        }))).expect(1).mount(&server).await;
    Mock::given(method("POST")).and(path("/v1/chat/completions"))
        .and(body_partial_json(json!({"model": "premium/model"})))
        .respond_with(ResponseTemplate::new(200).set_body_json(json!({
            "id": "answer-1", "object": "chat.completion", "model": "premium/model",
            "choices": [{"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"}]
        }))).expect(1).mount(&server).await;
    let output = runner
        .route("switchyard/direct")
        .unwrap()
        .execute(
            Request {
                llm_request: text_request(None, "Route this request"),
                metadata: Some(Metadata {
                    session_id: Some("session-1".into()),
                    ..Metadata::default()
                }),
                ..Request::default()
            },
            None,
        )
        .await?;
    assert_eq!(output.selected_model, "premium");
    let requests = server.received_requests().await.unwrap();
    let decision: Value = serde_json::from_slice(&requests[0].body)?;
    assert_eq!(
        decision["input"],
        json!([{"role": "user", "content": [{"type": "text", "text": "Route this request"}]}])
            .to_string()
    );
    server.verify().await;
    Ok(())
}

#[test]
fn direct_mode_rejects_invalid_configuration() {
    for (from, to, expected) in [
        (
            r#"judge = ["router"]"#,
            r#"judge = ["fast"]"#,
            "requires target kind decision",
        ),
        (
            "balanced = [\"balanced\"]\npremium = [\"premium\"]\n",
            "",
            "at least two options",
        ),
        (
            r#"default_target = "fast""#,
            r#"default_target = "any""#,
            "must be one of the options",
        ),
    ] {
        let source = direct_config("http://127.0.0.1:1").replace(from, to);
        let error = AlgorithmFactory::from_toml(&source)
            .and_then(|factory| factory.build())
            .err()
            .unwrap_or_else(|| panic!("expected a load error for {to}"))
            .to_string();
        assert!(error.contains(expected), "{to}: {error}");
    }
}