Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat: upgrade dev server to use Composite + Decision (#1010)
Graham King5a5bff2c49bd
diff --git a/dev-server/README b/dev-server/READMEindex a6a501c59..69f701ef0 100644--- a/dev-server/README+++ b/dev-server/README@@ -1,8 +1,9 @@# Switchyard dev server-```curl --insecure -s 'https://switchyard-dev.nvidia.com/v1/chat/completions' -H 'Content-Type: application/json' -d '{"messages":[{"content":"Hello from Switchyard","role":"user"}],"model":"switchyard/passthrough","max_tokens": 4096}'```+```curl --insecure -s 'https://switchyard-dev.nvidia.com/v1/chat/completions' -H 'Content-Type: application/json' -d '{"messages":[{"content":"Hello from Switchyard","role":"user"}],"model":"switchyard/auto","max_tokens": 4096}'```It has the following endpoints connected to Inference Hub:+. `switchyard/auto`: Composite. Decision model as classifier (Mercury Decide), then Stage on tool call signals.* `switchyard/noop`* `switchyard/passthrough` -> nvidia/openai/gpt-oss-20b* `switchyard/random` -> oss-20b (above) and nvidia_dynamo/deepseek-ai/deepseek-v4-flash-nvfp4-elbdiff --git a/dev-server/config.toml b/dev-server/config.tomlindex 2ce5a41c5..fee98672e 100644--- a/dev-server/config.toml+++ b/dev-server/config.toml@@ -1,24 +1,55 @@schema_version = 1-# Everything is on Inference Hub+# Clients[llm_clients.inference_hub]format = "openai_responses"base_url = "https://inference-api.nvidia.com/v1"api_key_env = "NVIDIA_API_KEY"-# The two models our algos use+[decision_clients.decisions]+format = "system_one"+endpoint = "https://openrouter.ai/api/alpha/decisions"+api_key_env = "OPENROUTER_API_KEY"+timeout_ms = 15000++# The three models our algos use[targets.capable]-id = "openai/openai/gpt-5.6-sol"+id = "openai/openai/gpt-6.1-sol"llm_client = "inference_hub"extra_body = { reasoning = { effort = "medium" } }[targets.efficient]-id = "openai/openai/gpt-5.6-luna"+id = "openai/openai/gpt-6-luna"llm_client = "inference_hub"extra_body = { reasoning = { effort = "medium" } }+[decision_targets.classifier]+id = "inception/mercury-decide:free"+decision_client = "decisions"++# Best route+[routes.auto]+id = "switchyard/auto"+type = "composite"++[routes.auto.classifier]+target = "classifier"+classify_trigger = "user_turn"+fail_open = false++[routes.auto.classifier.decision]+cutoff = 0.4+candidates = { a = "capable", b = "efficient" }+evidence = { description = "Candidate a is the larger reasoning-oriented coding and thinking model. It is intended for complex problems. Candidate b is the smaller fast coding model. It is intended for simple questions, simple local reads and edits, routine tests and straightforward fixes. These are qualitative deployment roles, not empirical success rates." }++[routes.auto.stage]+capable_target = "capable"+efficient_target = "efficient"+efficient_first_threshold = 0.5+capable_first_threshold = 0.0+# Basic routes[routes.random]diff --git a/dev-server/switchyard.service b/dev-server/switchyard.serviceindex 51962a9a1..dafefcf5d 100644--- a/dev-server/switchyard.service+++ b/dev-server/switchyard.service@@ -19,6 +19,7 @@ ExecStart=/usr/local/bin/switchyard-server \Restart=on-failureEnvironment="NVIDIA_API_KEY=<PUT-THE-KEY-HERE>"+Environment="OPENROUTER_API_KEY=<PUT-THE-OTHER-KEY-HERE>"PrivateDevices=yesProtectHome=yes