Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
build: Check in some dev server files (#289)
Graham King5fad270c5192
diff --git a/dev-server/README b/dev-server/READMEnew file mode 100644index 000000000..d6a0f0922--- /dev/null+++ b/dev-server/README@@ -0,0 +1,66 @@+# Switchyard dev server++```curl --insecure -s 'https://switchyard-dev.nvidia.com/v1/chat/completions' -H 'Content-Type: application/json' -d '{"messages":[{"content":"Hello from Switchyard","role":"user"}],"model":"switchyard/passthrough","max_tokens": 4096}'```++It has the following endpoints connected to Inference Hub:+* `switchyard/noop`+* `switchyard/passthrough` -> nvidia/openai/gpt-oss-20b+* `switchyard/random` -> oss-20b (above) and nvidia_dynamo/deepseek-ai/deepseek-v4-flash-nvfp4-elb+* `switchyard/stage` -> nvidia/zai-org/glm-5.2 and aws/anthropic/bedrock-claude-opus-4-8+* `switchyard/classifier` -> same models as stage++If you have an NVIDIA UNIX account and Silverfort I think you can ssh to it, ping for details. There's a `/home/README` with the infos once you're in.++The network has some kind of throttling. Not good for evals, but fine for poking around. Only accessible to internal NVIDIA folks for now.++## Files in this folder++- `switchyard.service`. The systemd unit. Goes in `/etc/systemd/system/switchyard.service`. Remember to `systemctl daemon-reload` if you change it. REMEMBER TO ADD THE API KEY.+- `config.toml`. The switchyard-server config. Goes in `/etc/switchyard/`.++## Details++This VM ping times maybe vary. `crates.io` appears throttled. Trying to `sudo` will trigger Silverfort.++### Manage++Switchyard is managed with systemd. Config is in `/etc/systemd/system/switchyard.service`.++- `sudo systemctl restart switchyard`+- `sudo systemctl status switchyard`+- `sudo systemctl stop switchyard`++### Logs++- `sudo journalctl -f -u switchyard`++### Update++#### git / binary++```+cd Switchyard/+git pull+export RUSTFLAGS="-Ctarget-cpu=native -Cforce-frame-pointers=yes"+cargo build --release+sudo cp target/release/switchyard-server /usr/local/bin/+sudo systemctl restart switchyard+```++The VM has weird network so have to retry sometimes. Check the status and logs after restarting (see 'Manage' above), because systemd won't tell you that it failed on startup.++#### config++Config is in `/etc/switchyard/config.toml`. Edit it in the repo though, then copy over.++#### TLS++Config and TLS key/cert are in `/etc/switchyard/`.++Re-generate:+```+openssl genpkey -algorithm Ed25519 -out key.pem+openssl req -new -x509 -key key.pem -days 1460 -out cert.pem -subj "/C=US/ST=CA/L=Santa Clara/O=LocalDev/OU=Switchyard/CN=switchyard-dev.nvidia.com"+cp {cert,key}.pem /etc/switchyard/+```+diff --git a/dev-server/config.toml b/dev-server/config.tomlnew file mode 100644index 000000000..321789081--- /dev/null+++ b/dev-server/config.toml@@ -0,0 +1,79 @@+schema_version = 1++# Everything is on Inference Hub++[llm_clients.inference_hub]+format = "openai_chat"+base_url = "https://inference-api.nvidia.com/v1"+api_key_env = "NVIDIA_API_KEY"++# The simple ones first++[targets.oss20b]+id = "nvidia/openai/gpt-oss-20b"+llm_client = "inference_hub"++[targets.dsflash]+id = "nvidia_dynamo/deepseek-ai/deepseek-v4-flash-nvfp4-elb"+llm_client = "inference_hub"++[routes.random]+id = "switchyard/random"+type = "random"+targets = ["oss20b", "dsflash"]++[routes.passthrough]+id = "switchyard/passthrough"+type = "passthrough"+target = "oss20b"++[routes.noop]+id = "switchyard/noop"+type = "noop"++# The two models are main algos use++[llm_clients.nvidia_anthropic]+format = "anthropic_messages"+base_url = "https://inference-api.nvidia.com"+api_key_env = "NVIDIA_API_KEY"++[targets.capable]+id = "aws/anthropic/bedrock-claude-opus-4-8"+llm_client = "nvidia_anthropic"++[targets.capable.extra_body.output_config]+effort = "medium"++[targets.efficient]+id = "nvidia/zai-org/glm-5.2"+llm_client = "inference_hub"++# Stage++[routes.stage]+id = "switchyard/stage"+type = "stage_router"+capable_target = "capable"+efficient_target = "efficient"+picker = "efficient_first"+confidence_threshold = 0.5+recent_turn_window = 3++[routes.stage.handoff_notes]+escalation_note = "[router-guidance] A weaker model was handling this task and showed signs of stalling, looping, or repeated errors on the preceding steps, so control was escalated to you, a stronger model. Re-examine the current state directly and do not simply repeat the previous approach."+only_on_wrong_signal_escalation = true++# Classifier++[routes.classifier]+id = "switchyard/classifier"+type = "llm_classifier"+mode = "capability"+classifier_target = "capable"+strong_target = "capable"+weak_target = "efficient"+base_threshold = 0.5+session_affinity = true+message_hash_fallback = true+diff --git a/dev-server/switchyard.service b/dev-server/switchyard.servicenew file mode 100644index 000000000..bc974610c--- /dev/null+++ b/dev-server/switchyard.service@@ -0,0 +1,30 @@+[Unit]+Description=Switchyard Server++[Service]+ExecStart=/usr/local/bin/switchyard-server -p 443 --config /etc/switchyard/config.toml --tls-key /etc/switchyard/key.pem --tls-cert /etc/switchyard/cert.pem+Restart=on-failure+Environment="NVIDIA_API_KEY=<PUT-THE-KEY-HERE>"+User=root+Group=root++# Start root, then harden/drop+NoNewPrivileges=yes+PrivateDevices=yes+ProtectSystem=strict+ProtectHome=yes+ReadOnlyPaths=/etc/switchyard+CapabilityBoundingSet=+AmbientCapabilities=+ProtectKernelTunables=yes+ProtectKernelModules=yes+ProtectControlGroups=yes++# Allow port 443+CapabilityBoundingSet=CAP_NET_BIND_SERVICE+AmbientCapabilities=CAP_NET_BIND_SERVICE++[Install]+WantedBy=multi-user.target++