Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat: upgrade dev server to use Composite + Decision (#1010)

Graham King5a5bff2c49bd
diff --git a/dev-server/README b/dev-server/README
index a6a501c59..69f701ef0 100644
--- a/dev-server/README
+++ b/dev-server/README
@@ -1,8 +1,9 @@
# Switchyard dev server
-```curl --insecure -s 'https://switchyard-dev.nvidia.com/v1/chat/completions' -H 'Content-Type: application/json' -d '{"messages":[{"content":"Hello from Switchyard","role":"user"}],"model":"switchyard/passthrough","max_tokens": 4096}'```
+```curl --insecure -s 'https://switchyard-dev.nvidia.com/v1/chat/completions' -H 'Content-Type: application/json' -d '{"messages":[{"content":"Hello from Switchyard","role":"user"}],"model":"switchyard/auto","max_tokens": 4096}'```
It has the following endpoints connected to Inference Hub:
+. `switchyard/auto`: Composite. Decision model as classifier (Mercury Decide), then Stage on tool call signals.
* `switchyard/noop`
* `switchyard/passthrough` -> nvidia/openai/gpt-oss-20b
* `switchyard/random` -> oss-20b (above) and nvidia_dynamo/deepseek-ai/deepseek-v4-flash-nvfp4-elb
diff --git a/dev-server/config.toml b/dev-server/config.toml
index 2ce5a41c5..fee98672e 100644
--- a/dev-server/config.toml
+++ b/dev-server/config.toml
@@ -1,24 +1,55 @@
schema_version = 1
-# Everything is on Inference Hub
+# Clients
[llm_clients.inference_hub]
format = "openai_responses"
base_url = "https://inference-api.nvidia.com/v1"
api_key_env = "NVIDIA_API_KEY"
-# The two models our algos use
+[decision_clients.decisions]
+format = "system_one"
+endpoint = "https://openrouter.ai/api/alpha/decisions"
+api_key_env = "OPENROUTER_API_KEY"
+timeout_ms = 15000
+
+# The three models our algos use
[targets.capable]
-id = "openai/openai/gpt-5.6-sol"
+id = "openai/openai/gpt-6.1-sol"
llm_client = "inference_hub"
extra_body = { reasoning = { effort = "medium" } }
[targets.efficient]
-id = "openai/openai/gpt-5.6-luna"
+id = "openai/openai/gpt-6-luna"
llm_client = "inference_hub"
extra_body = { reasoning = { effort = "medium" } }
+[decision_targets.classifier]
+id = "inception/mercury-decide:free"
+decision_client = "decisions"
+
+# Best route
+[routes.auto]
+id = "switchyard/auto"
+type = "composite"
+
+[routes.auto.classifier]
+target = "classifier"
+classify_trigger = "user_turn"
+fail_open = false
+
+[routes.auto.classifier.decision]
+cutoff = 0.4
+candidates = { a = "capable", b = "efficient" }
+evidence = { description = "Candidate a is the larger reasoning-oriented coding and thinking model. It is intended for complex problems. Candidate b is the smaller fast coding model. It is intended for simple questions, simple local reads and edits, routine tests and straightforward fixes. These are qualitative deployment roles, not empirical success rates." }
+
+[routes.auto.stage]
+capable_target = "capable"
+efficient_target = "efficient"
+efficient_first_threshold = 0.5
+capable_first_threshold = 0.0
+
# Basic routes
[routes.random]
diff --git a/dev-server/switchyard.service b/dev-server/switchyard.service
index 51962a9a1..dafefcf5d 100644
--- a/dev-server/switchyard.service
+++ b/dev-server/switchyard.service
@@ -19,6 +19,7 @@ ExecStart=/usr/local/bin/switchyard-server \
Restart=on-failure
Environment="NVIDIA_API_KEY=<PUT-THE-KEY-HERE>"
+Environment="OPENROUTER_API_KEY=<PUT-THE-OTHER-KEY-HERE>"
PrivateDevices=yes
ProtectHome=yes