Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
2.8 KB
Raw Blame
schema_version = 1

# Clients

[llm_clients.inference_hub]
format = "openai_responses"
base_url = "https://inference-api.nvidia.com/v1"
api_key_env = "NVIDIA_API_KEY"

[decision_clients.decisions]
format = "openai_decisions"
endpoint = "https://inference-api.nvidia.com/v1/decisions"
api_key_env = "NVIDIA_API_KEY"
timeout_ms = 5000

# The three models our algos use

[targets.capable]
id = "openai/openai/gpt-6.1-sol"
llm_client = "inference_hub"
extra_body = { reasoning = { effort = "medium" } }

[targets.efficient]
id = "openai/openai/gpt-6-luna"
llm_client = "inference_hub"
extra_body = { reasoning = { effort = "medium" } }

[decision_targets.classifier]
id = "openai/openai/gpt-6-luna"
decision_client = "decisions"

# Best route
[routes.auto]
id = "switchyard/auto"
type = "composite"

[routes.auto.classifier]
target = "classifier"
classify_trigger = "user_turn"
fail_open = false

[routes.auto.stage]
capable_target = "capable"
efficient_target = "efficient"
efficient_first_threshold = 0.5
capable_first_threshold = 0.2

# Basic routes

[routes.random]
id = "switchyard/random"
type = "random"
targets = ["capable", "efficient"]

[routes.passthrough]
id = "switchyard/passthrough"
type = "passthrough"
target = "efficient"

[routes.noop]
id = "switchyard/noop"
type = "noop"

# Stage

[routes.stage]
id = "switchyard/stage"
type = "stage_router"
capable_target = "capable"
efficient_target = "efficient"
picker = "efficient_first"
confidence_threshold = 0.5
recent_turn_window = 3

[routes.stage.handoff_notes]
escalation_note = "[router-guidance] A weaker model was handling this task and showed signs of stalling, looping, or repeated errors on the preceding steps, so control was escalated to you, a stronger model. Re-examine the current state directly and do not simply repeat the previous approach."
only_on_wrong_signal_escalation = true

# Classifier

[routes.classifier]
id = "switchyard/classifier"
type = "llm_classifier"
mode = "capability"
classifier_target = "capable"
strong_target = "capable"
weak_target = "efficient"
base_threshold = 0.5
classify_trigger = "new_session"
message_hash_fallback = true

# Advisor

[routes.advisor]
id = "switchyard/advisor"
type = "advisor"
executor_target = "efficient"
advisor_target = "capable"
max_reviews = 3
gate_stall_turns = 30
gate_min_tool_results = 3

# Codex "Approve for me" reviews

# With "Approve for me" on, Codex sends each review request to the model
# codex-auto-review, unless Codex is logged in with an OpenAI API key. Reviews
# go to the efficient model because Codex waits for each one before it acts.
# Codex asks for low reasoning effort in every review, and extra_body only adds
# an effort when the request has none, so reviews run at low effort.

[routes.codex_auto_review]
id = "codex-auto-review"
type = "passthrough"
target = "efficient"