Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Replace model IDs, addresses, and cloud authentication for your deployment.
schema_version = 1
# llama.cpp is active in this example. Discovery must be explicitly enabled.
[llm_clients.efficient]
format = "openai_chat"
base_url = "http://localhost:8080/v1"
context_discovery = "llama_cpp"
# For Ollama or vLLM, replace the active local client block with ONE block below.
# Also set context_window in routes.assistant to match the running service.
#
# Ollama:
# [llm_clients.efficient]
# format = "openai_chat"
# base_url = "http://localhost:11434/v1"
#
# vLLM:
# [llm_clients.efficient]
# format = "openai_chat"
# base_url = "http://localhost:8000/v1"
[llm_clients.capable]
format = "openai_chat"
base_url = "https://api.openai.com/v1"
# api_key_env = "OPENAI_API_KEY"
[targets.efficient]
# Use the llama.cpp alias, Ollama model tag, or vLLM served model name.
id = "efficient-model"
llm_client = "efficient"
[targets.capable]
id = "capable-model"
llm_client = "capable"
[routes.assistant]
id = "switchyard/vgr"
type = "vgr"
efficient_target = "efficient"
capable_target = "capable"
mode = "shadow"
# Explicit context_window takes precedence and skips discovery.
# Set this for Ollama/vLLM, or to set a deliberate limit with llama.cpp.
# context_window = 32768
# Hermes titles and approval checks bypass VGR verification.
[routes.auxiliary]
id = "switchyard/only-efficient"
type = "passthrough"
target = "efficient"