Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Tests for the dictionary-based libsy Python API."""
from collections.abc import AsyncIterator
from typing import Any
from uuid import UUID
import pytest
from switchyard.libsy import (
Algorithm,
ContextWindowExceededError,
CustomClassifierConfig,
DeescalationConfig,
EscalationClassifierConfig,
LlmClassifierConfig,
LlmResponse,
ModelCall,
OutcomeMetadata,
RoutingOutcome,
Step,
TaskClassifierConfig,
algorithms,
)
def request_body() -> dict[str, Any]:
return {
"model": "auto",
"messages": [
{
"role": "user",
"content": [{"type": "text", "text": "hello"}],
}
],
}
class EchoClient:
def __init__(self, model: str) -> None:
self.model = model
self.calls: list[dict[str, Any]] = []
async def call(self, request: dict[str, Any]) -> dict[str, Any]:
self.calls.append(request)
return {
"model": self.model,
"outputs": [
{
"role": "assistant",
"content": [{"type": "text", "text": self.model}],
"stop_reason": "end_turn",
}
],
}
async def ready(
response: LlmResponse.Agg | LlmResponse.Stream,
) -> LlmResponse.Agg | LlmResponse.Stream:
return response
async def aggregate(client: Any, request: dict[str, Any]) -> LlmResponse.Agg:
return LlmResponse.Agg(await client.call(request))
async def run_algorithm(
algorithm: Algorithm,
clients: dict[str, Any] | None = None,
*,
models: dict[str, list[str]] | None = None,
request: dict[str, Any] | None = None,
headers: dict[str, str] | None = None,
) -> tuple[str, dict[str, Any]]:
async def serve(call: ModelCall) -> LlmResponse.Agg:
for index, target in enumerate(call.models):
candidate_request = {**call.request, "model": target}
try:
return await aggregate((clients or {})[target], candidate_request)
except ContextWindowExceededError:
if index + 1 == len(call.models):
raise
raise AssertionError("model call has no candidates")
runtime_models = models if models is not None else {"any": list((clients or {}).keys())}
async for step in algorithm.run_stream(
request or request_body(), runtime_models, headers=headers
):
match step:
case Step.CallModel(call):
await call.respond(serve(call))
case Step.Done(outcome):
assert isinstance(outcome.metadata, OutcomeMetadata)
assert isinstance(outcome.candidate_requests, dict)
assert UUID(outcome.metadata.outcome_id).version == 7
if outcome.response is not None:
match outcome.response:
case LlmResponse.Agg(response):
return outcome.selected_model_ids[0], response
case LlmResponse.Stream(_):
raise AssertionError("test helper expected an aggregate response")
candidates = outcome.selected_model_ids
for index, target in enumerate(candidates):
candidate_request = {
**outcome.candidate_requests.get(target, outcome.request),
"model": target,
}
client = (clients or {})[target]
try:
response = await client.call(candidate_request)
except ContextWindowExceededError:
if index + 1 == len(candidates):
raise
else:
return outcome.selected_model_ids[0], response
raise AssertionError("algorithm stream ended without an outcome")
async def test_random_streams_complex_steps_and_accepts_a_dictionary_response() -> None:
client = EchoClient("fast")
algorithm = algorithms.random()
outcome: RoutingOutcome | None = None
variants: list[str] = []
models = {"any": ["fast"]}
async for step in algorithm.run_stream(request_body(), models):
match step:
case Step.Done(done):
variants.append("done")
outcome = done
assert variants == ["done"]
assert outcome is not None
assert outcome.selected_model_ids == ["fast"]
assert outcome.response is None
assert outcome.metadata is not None
assert UUID(outcome.metadata.outcome_id).version == 7
assert outcome.metadata.algorithm == "random"
assert outcome.metadata.evidence is None
response = await client.call(outcome.request)
assert client.calls[0]["model"] == "fast"
assert client.calls[0]["messages"][0]["content"] == [{"type": "text", "text": "hello"}]
assert response["model"] == "fast"
assert response["outputs"][0]["content"] == [{"type": "text", "text": "fast"}]
async def test_routing_call_accepts_a_streamed_response() -> None:
async def events() -> AsyncIterator[dict[str, object]]:
for chunk in [
{"MessageStart": {"id": "response-1", "model": "judge"}},
{"TextDelta": {"index": 0, "text": '{"target":"efficient"}'}},
{"MessageStop": {"reason": "end_turn"}},
]:
yield {"preservation": None, "normalized": [chunk]}
schema = {
"type": "object",
"additionalProperties": False,
"required": ["target"],
"properties": {"target": {"type": "string", "enum": ["capable", "efficient"]}},
}
algorithm = algorithms.llm_classifier(
LlmClassifierConfig.custom(
default_target="capable",
config=CustomClassifierConfig("Choose a target.", schema, "/target"),
)
)
outcome: RoutingOutcome | None = None
models = {
"judge": ["judge"],
"capable": ["model-a"],
"efficient": ["model-b"],
"any": ["model-a", "model-b"],
}
async for step in algorithm.run_stream(request_body(), models):
match step:
case Step.CallModel(call):
await call.respond(ready(LlmResponse.Stream(events())))
case Step.Done(done):
outcome = done
assert outcome is not None
assert outcome.selected_model_ids == ["model-b", "model-a"]
async def test_classifier_config_accepts_a_prompt_override() -> None:
"""Verify that a configured classifier prompt is rendered for the judge."""
class JudgeClient(EchoClient):
async def call(self, request: dict[str, Any]) -> dict[str, Any]:
self.calls.append(request)
return {
"model": self.model,
"outputs": [
{
"role": "assistant",
"content": [
{
"type": "text",
"text": (
'{"crux":"bounded task","primary_rule":"SUP-1",'
'"capability_boundary":"supported","p_solve":0.9}'
),
}
],
"stop_reason": "end_turn",
}
],
}
judge = JudgeClient("judge")
weak = EchoClient("weak")
algorithm = algorithms.llm_classifier(
LlmClassifierConfig.capability(
config=TaskClassifierConfig(
0.5,
threshold_step=0.1,
prompt="Custom capability rubric.",
),
),
)
outcome: RoutingOutcome | None = None
models={
"judge": ["judge"],
"efficient": ["weak"],
"capable": ["strong"],
"any": ["weak", "strong"],
}
async for step in algorithm.run_stream(request_body(), models):
match step:
case Step.CallModel(call):
await call.respond(aggregate(judge, call.request))
case Step.Done(done):
outcome = done
assert outcome is not None
assert outcome.selected_model_ids[0] == "weak"
assert outcome.metadata is not None
assert outcome.metadata.evidence == {
"source": "llm-classifier",
"score": pytest.approx(0.9),
"threshold": pytest.approx(0.5),
}
response = await weak.call(outcome.request)
prompt = judge.calls[0]["instructions"][0]["content"][0]["text"]
assert prompt == "Custom capability rubric."
assert judge.calls[0]["output"]["response_format"]["json_schema"]["schema"]["properties"][
"p_solve"
]
assert response["model"] == "weak"
def test_escalation_accepts_optional_deescalation_config() -> None:
config = EscalationClassifierConfig(
deescalation=DeescalationConfig(
strong_min_calls=3,
confirmations=2,
strong_max_calls=6,
weak_cooldown_calls=8,
)
)
algorithm = LlmClassifierConfig.escalation(config=config)
assert isinstance(algorithms.llm_classifier(algorithm), Algorithm)
async def test_custom_classifier_routes_across_named_targets() -> None:
class JudgeClient(EchoClient):
async def call(self, request: dict[str, Any]) -> dict[str, Any]:
self.calls.append(request)
return {
"model": self.model,
"outputs": [
{
"role": "assistant",
"content": [{"type": "text", "text": '{"target":"efficient"}'}],
"stop_reason": "end_turn",
}
],
}
schema = {
"type": "object",
"additionalProperties": False,
"required": ["target"],
"properties": {"target": {"type": "string", "enum": ["capable", "efficient"]}},
}
algorithm = algorithms.llm_classifier(
LlmClassifierConfig.custom(
default_target="capable",
config=CustomClassifierConfig("Choose a target.", schema, "/target"),
)
)
_, response = await run_algorithm(
algorithm,
{
"judge": JudgeClient("judge"),
"model-a": EchoClient("model-a"),
"model-b": EchoClient("model-b"),
"model-c": EchoClient("model-c"),
},
models={
"judge": ["judge"],
"capable": ["model-a", "model-c"],
"efficient": ["model-b"],
"any": ["model-a", "model-b", "model-c"],
},
)
assert response["model"] == "model-b"
async def test_classifier_config_accepts_json_object_output() -> None:
"""Verify that Python can select JSON Object mode for a classifier judge."""
class JudgeClient(EchoClient):
async def call(self, request: dict[str, Any]) -> dict[str, Any]:
self.calls.append(request)
return {
"model": self.model,
"outputs": [
{
"role": "assistant",
"content": [
{
"type": "text",
"text": (
'{"crux":"bounded task","primary_rule":"SUP-1",'
'"capability_boundary":"supported","p_solve":0.9}'
),
}
],
"stop_reason": "end_turn",
}
],
}
judge = JudgeClient("judge")
weak = EchoClient("weak")
algorithm = algorithms.llm_task_classifier(
config=TaskClassifierConfig(0.5, response_format_type="json_object"),
)
_, response = await run_algorithm(
algorithm,
{
"judge": judge,
"weak": weak,
"strong": EchoClient("strong"),
},
models={
"judge": ["judge"],
"efficient": ["weak"],
"capable": ["strong"],
"any": ["weak", "strong"],
},
)
assert judge.calls[0]["output"]["response_format"] == {"type": "json_object"}
prompt = judge.calls[0]["instructions"][0]["content"][0]["text"]
assert "JSON Schema" in prompt
assert '"p_solve"' in prompt
assert response["model"] == "weak"
def test_classifier_config_rejects_unknown_response_format() -> None:
invalid_response_format: Any = "yaml"
with pytest.raises(
ValueError,
match="response_format_type must be 'json_schema' or 'json_object'",
):
TaskClassifierConfig(0.5, response_format_type=invalid_response_format)
async def test_random_weights_and_seed_are_reproducible() -> None:
def algorithm():
return algorithms.random(
weights=[1, 3],
seed=42,
)
first_router = algorithm()
second_router = algorithm()
clients = {"fast": EchoClient("fast"), "capable": EchoClient("capable")}
first = [(await run_algorithm(first_router, clients))[1]["model"] for _ in range(100)]
second = [(await run_algorithm(second_router, clients))[1]["model"] for _ in range(100)]
assert first == second
assert 65 <= second.count("capable") <= 85
def test_random_rejects_invalid_weights() -> None:
with pytest.raises(ValueError, match="finite and nonnegative"):
algorithms.random(weights=[-1])
async def test_noop_needs_no_client() -> None:
selected_model, response = await run_algorithm(algorithms.noop())
assert selected_model == "auto"
assert response["outputs"][0]["content"] == [{"type": "text", "text": "OK"}]
@pytest.mark.parametrize(("method", "base"), [("model_dump", object), ("to_dict", dict)])
def test_conversion_getter_error_propagates(method: str, base: type) -> None:
error = RuntimeError("conversion getter failed")
def fail(self: Any) -> Any:
raise error
request_type = type("Request", (base,), {method: property(fail)})
request = request_type(request_body()) if base is dict else request_type()
with pytest.raises(RuntimeError) as caught:
algorithms.noop().run_stream(request, {"any": ["fixture"]})
assert caught.value is error
@pytest.mark.parametrize(
("headers", "message"),
[
({"invalid header": "value"}, "invalid HTTP header name"),
({"x-valid": "invalid\nvalue"}, "failed to parse header value"),
],
)
def test_algorithm_rejects_invalid_headers(headers: dict[str, str], message: str) -> None:
with pytest.raises(ValueError, match=message):
algorithms.noop().run_stream(request_body(), {}, headers=headers)
async def test_algorithm_accepts_case_insensitive_duplicate_names() -> None:
selected_model, _ = await run_algorithm(
algorithms.noop(), headers={"X-Unused": "first", "x-unused": "second"}
)
assert selected_model == "auto"
def test_algorithm_rejects_header_map_capacity_overflow() -> None:
headers = {f"x-header-{index}": "value" for index in range(32_769)}
with pytest.raises(ValueError, match="max size reached"):
algorithms.noop().run_stream(request_body(), {}, headers=headers)
def test_algorithm_exposes_only_streaming_execution() -> None:
algorithm = algorithms.noop()
assert callable(algorithm.run_stream)
assert not hasattr(algorithm, "run")
def test_invalid_request_is_rejected_at_the_boundary() -> None:
algorithm = algorithms.random()
with pytest.raises(ValueError, match="unknown variant"):
algorithm.run_stream(
{
"model": "auto",
"messages": [{"role": "invalid", "content": []}],
},
{"any": ["fast"]},
)
async def test_context_window_failure_falls_back_to_the_next_model() -> None:
class OverflowClient:
def __init__(self) -> None:
self.calls: list[dict[str, Any]] = []
async def call(self, request: dict[str, Any]) -> dict[str, Any]:
self.calls.append(request)
raise ContextWindowExceededError("request exceeds context window")
overflow = OverflowClient()
algorithm = algorithms.stage_router(
picker="efficient_first",
confidence_threshold=0.5,
efficient_system_prompt="Use the efficient tier.",
)
selected_model, response = await run_algorithm(
algorithm,
{"fast": overflow, "strong": EchoClient("strong")},
models={
"efficient": ["fast"],
"capable": ["strong"],
"any": ["fast", "strong"],
},
)
assert selected_model == "fast"
assert response["model"] == "strong"
assert overflow.calls[0]["instructions"] == [
{
"role": "system",
"content": [{"type": "text", "text": "Use the efficient tier."}],
}
]
async def test_escalation_falls_back_on_first_stream_context_overflow() -> None:
async def overflow() -> AsyncIterator[dict[str, object]]:
if False:
yield {}
raise ContextWindowExceededError("weak first-event context overflow")
algorithm = algorithms.llm_classifier(
LlmClassifierConfig.escalation(
config=EscalationClassifierConfig(confirmations=1),
)
)
calls: list[list[str]] = []
outcome: RoutingOutcome | None = None
models = {
"judge": ["judge"],
"efficient": ["weak"],
"capable": ["strong"],
"any": ["weak", "strong"],
}
async for step in algorithm.run_stream(
{**request_body(), "stream": True}, models
):
match step:
case Step.CallModel(call):
calls.append(call.models)
await call.respond(ready(LlmResponse.Stream(overflow())))
case Step.Done(done):
outcome = done
assert calls == [["weak"]]
assert outcome is not None
assert outcome.selected_model_ids == ["strong", "weak"]
assert outcome.response is None
assert outcome.metadata is not None
assert outcome.metadata.evidence == {
"source": "fallback",
"reason_code": "context_window",
}
async def test_stage_router_applies_additive_tool_semantics() -> None:
"""Verify that configured mutation semantics reach stage-router scoring."""
algorithm = algorithms.stage_router(
picker="capable_first",
confidence_threshold=0.3,
tool_semantics={
"observe": ["KB_search"],
"mutate": ["send_payment_request"],
"plan": ["create_research_plan"],
"new": ["send_message_to_user"],
},
)
selected_model, _ = await run_algorithm(
algorithm,
{"strong": EchoClient("strong"), "fast": EchoClient("fast")},
models={
"efficient": ["fast"],
"capable": ["strong"],
"any": ["fast", "strong"],
},
request={
"model": "auto",
"messages": [
{
"role": "user",
"content": [{"type": "text", "text": "pay the balance"}],
},
{
"role": "assistant",
"content": [
{
"type": "tool_call",
"id": "call_1",
"name": "send_payment_request",
"arguments": {},
}
],
},
{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_call_id": "call_1",
"content": [{"type": "text", "text": "payment sent"}],
"is_error": None,
}
],
},
],
},
)
assert selected_model == "fast"
def test_stage_router_rejects_unknown_tool_semantics_category() -> None:
with pytest.raises(ValueError, match="unknown tool_semantics category"):
algorithms.stage_router(
picker="efficient_first",
confidence_threshold=0.5,
tool_semantics={"complete": ["end_conversation"]},
)