Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package switchyard
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"runtime"
"slices"
"strings"
"sync"
"testing"
"time"
)
const stage = `"capable_target":"strong","efficient_target":"weak","confidence_threshold":0.5`
const composite = `{"type":"composite","classifier":{"target":"judge","base_threshold":0.5,"classify_trigger":"user_turn","fail_open":false},"stage":{` + stage + `}}`
const decision = `{"type":"composite","classifier":{"target":"judge","classify_trigger":"user_turn","fail_open":false,"decision":{"cutoff":0.5}},"stage":{` + stage + `}}`
const judgeAnswer = `{"model":"provider-model","outputs":[{"role":"assistant","content":[{"type":"text","text":"{\"crux\":\"bounded task\",\"primary_rule\":\"SUP-1\",\"capability_boundary\":\"supported\",\"p_solve\":0.9}"}],"stop_reason":"end_turn"}]}`
func newRouter(t *testing.T, algorithm string) *Router {
t.Helper()
router, err := New(testConfig(algorithm))
if err != nil {
t.Fatal(err)
}
t.Cleanup(router.Close)
return router
}
func input() Input {
return Input{
Request: json.RawMessage(`{"messages":[{"role":"user","content":[{"type":"text","text":"Fix this small bug"}]}]}`),
Models: map[string][]string{
"any": {"strong-model", "weak-model"}, "capable": {"strong-model"},
"efficient": {"weak-model"}, "judge": {"judge-model"},
},
}
}
// Completed outcomes retain fallback requests while absent requests remain optional.
func TestOutcomeFallbackRequest(t *testing.T) {
request := input().Request
for _, tc := range []struct {
name, field string
present bool
}{
{"present", `"fallback_request":` + string(request), true},
{"missing", "", false},
{"null", `"fallback_request":null`, false},
} {
t.Run(tc.name, func(t *testing.T) {
var outcome Outcome
data := []byte(`{` + tc.field + `}`)
if err := json.Unmarshal(data, &outcome); err != nil {
t.Fatal(err)
}
if present := outcome.FallbackRequest != nil; present != tc.present {
t.Fatalf("fallback request present=%v, want %v", present, tc.present)
}
if tc.present && string(*outcome.FallbackRequest) != string(request) {
t.Fatalf("fallback request=%s, want %s", *outcome.FallbackRequest, request)
}
})
}
}
// Stage's fallback keeps its execution prompt instead of reusing the planning prompt.
func TestCandidateRequestPreservesFallbackPrompt(t *testing.T) {
router := newRouter(t, `{"type":"stage_router","picker":"capable_first",`+stage+`,"capable_system_prompt":"Plan the change","efficient_system_prompt":"Execute the change"}`)
outcome, err := router.Route(context.Background(), input(), nil)
if err != nil {
t.Fatal(err)
}
wire, err := EncodeRequest("openai_chat", outcome.CandidateRequests["weak-model"], "weak-model")
if err != nil {
t.Fatal(err)
}
var encoded struct {
Messages []struct{ Role, Content string }
}
if err := json.Unmarshal(wire, &encoded); err != nil {
t.Fatal(err)
}
if len(encoded.Messages) == 0 || encoded.Messages[0].Role != "system" || encoded.Messages[0].Content != "Execute the change" {
t.Fatalf("fallback request=%s, want execution prompt", wire)
}
}
// Selection retains eligible targets and Stage's prepared request across the codecs.
func TestSelectionAndTranslation(t *testing.T) {
type chatRequest struct {
Model string
Store *bool
Messages []struct{ Role, Content string }
}
for _, tc := range []struct {
name, algorithm, expected string
failed bool
}{
{"random", `{"type":"random","targets":["strong","weak"]}`, "weak-model", false},
{"stage", `{"type":"stage_router","picker":"efficient_first",` + stage + `}`, "weak-model", false},
{"escalation", `{"type":"stage_router","picker":"efficient_first",` + stage + `,"handoff_notes":{"escalation_note":"Review the failure"}}`, "strong-model", true},
} {
t.Run(tc.name, func(t *testing.T) {
router := newRouter(t, tc.algorithm)
request := input()
if tc.name == "random" {
request.Models["any"] = []string{"weak-model"}
}
if tc.failed {
request.Request = json.RawMessage(`{"messages":[{"role":"assistant","content":[{"type":"tool_call","id":"c","name":"bash","arguments":{}}]},{"role":"tool","content":[{"type":"tool_result","tool_call_id":"c","content":[{"type":"text","text":"Out of memory"}],"is_error":true}]}]}`)
}
outcome, err := router.Route(context.Background(), request, nil)
if err != nil || len(outcome.SelectedModelIDs) == 0 || outcome.SelectedModelIDs[0] != tc.expected {
t.Fatalf("outcome=%+v err=%v", outcome, err)
}
wire, err := EncodeRequest("openai_chat", outcome.Request, tc.expected)
if err != nil {
t.Fatal(err)
}
var prepared chatRequest
if err := json.Unmarshal(wire, &prepared); err != nil || prepared.Model != tc.expected {
t.Fatalf("prepared request=%s err=%v", wire, err)
}
if tc.failed {
if len(prepared.Messages) == 0 {
t.Fatal("missing handoff message")
}
last := prepared.Messages[len(prepared.Messages)-1]
if last.Role != "user" || last.Content != "Review the failure" {
t.Fatalf("handoff message=%+v", last)
}
}
})
}
request, err := DecodeRequest("openai_chat", json.RawMessage(`{"model":"route","messages":[{"role":"user","content":"hello"}],"store":false}`))
if err != nil {
t.Fatal(err)
}
wire, err := EncodeRequest("openai_chat", request, "actual-target")
if err != nil {
t.Fatal(err)
}
var encoded chatRequest
if err := json.Unmarshal(wire, &encoded); err != nil || encoded.Model != "actual-target" || encoded.Store == nil || *encoded.Store {
t.Fatalf("wire=%s err=%v", wire, err)
}
response, err := DecodeResponse("openai_chat", json.RawMessage(`{"choices":[{"index":0,"message":{"role":"assistant","content":"hello"},"finish_reason":"stop"}]}`))
if err != nil {
t.Fatal(err)
}
var decoded struct {
Outputs []struct {
Content []struct{ Text string }
}
}
if err := json.Unmarshal(response, &decoded); err != nil || len(decoded.Outputs) != 1 || len(decoded.Outputs[0].Content) != 1 || decoded.Outputs[0].Content[0].Text != "hello" {
t.Fatalf("response=%s err=%v", response, err)
}
}
// Both judge protocols use Go's client; missing clients and host failures remain errors.
func TestCompositeHostCalls(t *testing.T) {
for _, tc := range []struct {
name, algorithm, kind, selected, response, invalidResponse string
}{
{"model", composite, "call_model", "weak-model", judgeAnswer, `{"outputs":"wrong"}`},
{"decision", decision, "call_decision", "strong-model", `{"answers":{"route":{"value":{"type":"choice","data":{"selected":"capable","probabilities":{"capable":0.9,"efficient":0.1}}}}}}`, `{"answers":"wrong"}`},
} {
t.Run(tc.name, func(t *testing.T) {
router := newRouter(t, tc.algorithm)
calls := 0
outcome, err := router.Route(context.Background(), input(), func(_ context.Context, work Work) (Reply, error) {
calls++
isJudge := work.Model == "judge-model"
if tc.kind == "call_model" {
isJudge = slices.Equal(work.Models, []string{"judge-model"})
}
if work.Type != tc.kind || !isJudge || !json.Valid(work.Request) {
return Reply{}, fmt.Errorf("unexpected work: %+v", work)
}
return Reply{Response: json.RawMessage(tc.response)}, nil
})
if err != nil || calls != 1 || len(outcome.SelectedModelIDs) == 0 || outcome.SelectedModelIDs[0] != tc.selected {
t.Fatalf("calls=%d outcome=%+v err=%v", calls, outcome, err)
}
if _, err := router.Route(context.Background(), input(), nil); err == nil {
t.Fatal("missing host client succeeded")
}
_, err = router.Route(context.Background(), input(), func(context.Context, Work) (Reply, error) {
return Reply{}, errors.New("host unavailable")
})
if err == nil || !strings.Contains(err.Error(), "host unavailable") {
t.Fatalf("host failure returned %v", err)
}
_, err = router.Route(context.Background(), input(), func(context.Context, Work) (Reply, error) {
return Reply{Response: json.RawMessage(tc.invalidResponse)}, nil
})
if err == nil {
t.Fatal("invalid host response succeeded")
}
})
}
}
// Host failures remain subject to the algorithm's configured recovery policy.
func TestHostFailureUsesAlgorithmPolicy(t *testing.T) {
for _, algorithm := range []string{composite, decision} {
router := newRouter(t, strings.Replace(algorithm, `"fail_open":false`, `"fail_open":true`, 1))
calls := 0
outcome, err := router.Route(context.Background(), input(), func(context.Context, Work) (Reply, error) {
calls++
return Reply{}, errors.New("judge unavailable")
})
if err != nil || calls != 1 || !slices.Equal(outcome.SelectedModelIDs, []string{"weak-model", "strong-model"}) {
t.Fatalf("calls=%d outcome=%+v err=%v", calls, outcome, err)
}
}
}
// An aborted host call stops routing even when the algorithm recovers from host errors.
func TestHostAbortStopsRouting(t *testing.T) {
if Abort(nil) != nil {
t.Fatal("nil error should stay nil")
}
errBudget := errors.New("budget exhausted")
refused := &fs.PathError{Op: "route", Path: "judge", Err: errBudget}
original := fmt.Errorf("judge call refused: %w", refused)
for _, algorithm := range []string{composite, decision} {
router := newRouter(t, strings.Replace(algorithm, `"fail_open":false`, `"fail_open":true`, 1))
calls := 0
var hostCtx context.Context
_, err := router.Route(context.Background(), input(), func(ctx context.Context, _ Work) (Reply, error) {
calls++
hostCtx = ctx
return Reply{}, Abort(original)
})
var typed *fs.PathError
if calls != 1 || err != original || !errors.Is(err, errBudget) || !errors.As(err, &typed) || typed != refused {
t.Fatalf("calls=%d err=%v", calls, err)
}
if hostCtx.Err() != context.Canceled {
t.Fatal("abort did not cancel host context")
}
}
}
// Cancellation releases routing work even if a host ignores its context and replies later.
func TestCancellationAndLateHostReply(t *testing.T) {
router := newRouter(t, composite)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
started, release, returned := make(chan struct{}), make(chan struct{}), make(chan struct{})
releaseHost := sync.OnceFunc(func() { close(release) })
defer releaseHost()
finished := make(chan error, 1)
go func() {
_, err := router.Route(ctx, input(), func(context.Context, Work) (Reply, error) {
defer close(returned)
router.Close()
close(started)
<-release
return Reply{Response: json.RawMessage(judgeAnswer)}, nil
})
finished <- err
}()
select {
case <-started:
case <-time.After(5 * time.Second):
t.Fatal("host callback could not close its router")
}
runtime.GC()
cancel()
select {
case err := <-finished:
if !errors.Is(err, context.Canceled) {
t.Fatalf("cancellation returned %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("cancellation waited for host code")
}
runtime.GC()
// Release after Route has freed its run, then let the late Go reply drain.
releaseHost()
select {
case <-returned:
case <-time.After(5 * time.Second):
t.Fatal("late host callback did not finish")
}
}
// Concurrent requests keep eligible pools separate; router copies share closed state.
func TestConcurrentRoutesAndClose(t *testing.T) {
router := newRouter(t, `{"type":"random","targets":["weak","strong"],"seed":7}`)
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := router.Route(ctx, input(), nil); !errors.Is(err, context.Canceled) {
t.Fatalf("cancelled context returned %v", err)
}
results := make(chan error, 24)
for i := 0; i < cap(results); i++ {
go func(i int) {
request := input()
model := []string{"weak-model", "strong-model"}[i%2]
request.Models = map[string][]string{"any": {model}}
outcome, err := router.Route(context.Background(), request, nil)
if err == nil && (len(outcome.SelectedModelIDs) != 1 || outcome.SelectedModelIDs[0] != model) {
err = fmt.Errorf("expected %s, got %v", model, outcome.SelectedModelIDs)
}
results <- err
}(i)
}
for i := 0; i < cap(results); i++ {
select {
case err := <-results:
if err != nil {
t.Fatal(err)
}
case <-time.After(5 * time.Second):
t.Fatal("concurrent route did not finish")
}
}
copy := *router
for _, closed := range []*Router{©, router, new(Router)} {
closed.Close()
closed.Close()
if _, err := closed.Route(context.Background(), input(), nil); err == nil {
t.Fatal("closed router accepted a request")
}
}
racing := newRouter(t, `{"type":"random","targets":["weak"]}`)
racingCopy, start := *racing, make(chan struct{})
go func() {
<-start
request := input()
request.Models["any"] = []string{"weak-model"}
outcome, err := racing.Route(context.Background(), request, nil)
if err != nil && err.Error() == "Switchyard router is closed" {
err = nil
} else if err == nil && (len(outcome.SelectedModelIDs) != 1 || outcome.SelectedModelIDs[0] != "weak-model") {
err = fmt.Errorf("close race selected %v", outcome.SelectedModelIDs)
}
results <- err
}()
go func() { <-start; racingCopy.Close(); results <- nil }()
close(start)
for i := 0; i < 2; i++ {
select {
case err := <-results:
if err != nil {
t.Fatal(err)
}
case <-time.After(5 * time.Second):
t.Fatal("close race did not finish")
}
}
runtime.GC()
}
// Served-target attribution and agent/session metadata survive the bridge.
func TestReplyAttributionAndMetadata(t *testing.T) {
router := newRouter(t, `{"type":"advisor","executor_target":"strong","advisor_target":"judge","gate_trigger":"pattern","gate_trigger_pattern":"NEVER"}`)
request := input()
request.Models["efficient"] = []string{"strong-model", "weak-model"}
outcome, err := router.Route(context.Background(), request, func(_ context.Context, work Work) (Reply, error) {
if len(work.Models) != 2 {
return Reply{}, fmt.Errorf("missing fallback: %+v", work)
}
return Reply{Response: json.RawMessage(judgeAnswer), ServedModel: "weak-model"}, nil
})
if err != nil || len(outcome.SelectedModelIDs) != 1 || outcome.SelectedModelIDs[0] != "weak-model" || outcome.Response == nil {
t.Fatalf("outcome=%+v err=%v", outcome, err)
}
delegated := newRouter(t, `{"type":"passthrough","target":"strong","subagents":{"type":"passthrough","target":"weak"}}`)
for _, child := range []bool{false, true} {
request := input()
request.Models = map[string][]string{"any": {"strong-model"}}
request.SubagentModels = map[string][]string{"any": {"weak-model"}}
request.Metadata = &RequestMetadata{SessionID: "session", AgentID: "child", IsSubagent: true, IsDelegatedWork: child}
expected := "strong-model"
if child {
expected = "weak-model"
}
outcome, err := delegated.Route(context.Background(), request, nil)
if err != nil || len(outcome.SelectedModelIDs) != 1 || outcome.SelectedModelIDs[0] != expected {
t.Fatalf("delegated=%v outcome=%+v err=%v", child, outcome, err)
}
}
request = input()
request.SubagentModels = map[string][]string{}
request.Metadata = &RequestMetadata{IsSubagent: true, IsDelegatedWork: true}
encoded, err := json.Marshal(request)
if err != nil {
t.Fatal(err)
}
var envelope struct {
SubagentModels map[string][]string `json:"subagent_models"`
}
if err := json.Unmarshal(encoded, &envelope); err != nil || envelope.SubagentModels == nil || len(envelope.SubagentModels) != 0 {
t.Fatalf("explicit empty child pool was lost: %s err=%v", encoded, err)
}
if _, err := delegated.Route(context.Background(), request, nil); err == nil {
t.Fatal("empty child pool selected a parent model")
}
// Both agents need active holds so a final request tests cleanup of the sibling's hold.
plan := newRouter(t, `{"type":"stage_router","capable_target":"strong","efficient_target":"weak","picker":"capable_first","confidence_threshold":0,"efficient_hold_turns":-1}`)
for _, tc := range []struct {
agent string
child, final, edited bool
expected string
}{
{"a", true, false, false, "strong-model"},
{"a", true, false, true, "weak-model"},
{"b", true, false, false, "strong-model"},
{"b", true, false, true, "weak-model"},
{"root", false, false, false, "strong-model"},
{"a", true, true, false, "weak-model"},
{"b", true, false, false, "strong-model"},
} {
request := input()
request.Request = json.RawMessage(`{}`)
if tc.edited {
request.Request = json.RawMessage(`{"messages":[{"role":"assistant","content":[{"type":"tool_call","id":"edit","name":"apply_patch","arguments":{}}]},{"role":"tool","content":[{"type":"tool_result","tool_call_id":"edit","content":[{"type":"text","text":"Applied patch"}],"is_error":false}]}]}`)
}
request.Metadata = &RequestMetadata{SessionID: "session", AgentID: tc.agent, IsSubagent: tc.child, SessionFinal: &tc.final}
outcome, err := plan.Route(context.Background(), request, nil)
if err != nil || len(outcome.SelectedModelIDs) == 0 || outcome.SelectedModelIDs[0] != tc.expected {
t.Fatalf("agent=%s final=%v outcome=%+v err=%v", tc.agent, tc.final, outcome, err)
}
}
}