Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(examples): update LiteLLM for run stream (#412)

nachiketb-nvidiae9767d642461
diff --git a/crates/switchyard-py/src/errors.rs b/crates/switchyard-py/src/errors.rs
index 9e0beb775..8f5e44e47 100644
--- a/crates/switchyard-py/src/errors.rs
+++ b/crates/switchyard-py/src/errors.rs
@@ -8,6 +8,7 @@ use pyo3::exceptions::PyRuntimeError;
use pyo3::prelude::*;
create_exception!(_switchyard_rust, LibsyError, PyRuntimeError);
+create_exception!(_switchyard_rust, ContextWindowExceededError, PyRuntimeError);
/// Converts libsy execution failures into one stable Python exception.
pub(crate) fn py_libsy_error(error: impl std::fmt::Display) -> PyErr {
@@ -15,5 +16,9 @@ pub(crate) fn py_libsy_error(error: impl std::fmt::Display) -> PyErr {
}
pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
- module.add("LibsyError", module.py().get_type::<LibsyError>())
+ module.add("LibsyError", module.py().get_type::<LibsyError>())?;
+ module.add(
+ "ContextWindowExceededError",
+ module.py().get_type::<ContextWindowExceededError>(),
+ )
}
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index ee2b70fd9..ce303d5e6 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -21,7 +21,7 @@ use switchyard_protocol::{
};
use tokio::sync::Mutex;
-use crate::errors::py_libsy_error;
+use crate::errors::{ContextWindowExceededError, py_libsy_error};
use crate::py_serde::{from_python, to_python};
/// Convert Python-owned headers into the request metadata expected by libsy.
@@ -250,8 +250,15 @@ impl PyModelCall {
}
let call = self.take()?;
let target = call.decision.selected_model_id().clone();
- let source = LlmClientError::Ffi {
- source: Box::new(PyErr::from_value(error.clone())),
+ let source = if error.is_instance_of::<ContextWindowExceededError>() {
+ LlmClientError::ContextWindowExceeded {
+ model: target.clone(),
+ message: error.str()?.to_string_lossy().into_owned(),
+ }
+ } else {
+ LlmClientError::Ffi {
+ source: Box::new(PyErr::from_value(error.clone())),
+ }
};
call.respond(Err(RustLibsyError::client_call(target, source)))
.map_err(py_libsy_error)
@@ -505,6 +512,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
&libsy_module
)?)?;
libsy_module.add_function(wrap_pyfunction!(stage_router_algorithm, &libsy_module)?)?;
+ libsy_module.add(
+ "ContextWindowExceededError",
+ module.getattr("ContextWindowExceededError")?,
+ )?;
libsy_module.add("LibsyError", module.getattr("LibsyError")?)?;
module.add_submodule(&libsy_module)?;
Ok(())
diff --git a/examples/experimental/litellm/README.md b/examples/experimental/litellm/README.md
index 1f2294e9a..836cca915 100644
--- a/examples/experimental/litellm/README.md
+++ b/examples/experimental/litellm/README.md
@@ -11,8 +11,8 @@ point for your own application.
LiteLLM provides the OpenAI-compatible gateway, model aliases, and OpenRouter
provider integration. Switchyard's Stage router makes the routing decision from
the coding agent's recent tool history.
-`LiteLLMSyClient` connects Switchyard's normalized libsy requests to the
-selected alias through LiteLLM's asynchronous Completion API and the
+`LiteLLMSyClient` connects Switchyard's normalized libsy requests to their
+selected aliases through LiteLLM's asynchronous Completion API and the
Dockerized gateway. Together, they let an application keep routing policy in
Switchyard while LiteLLM owns model access and sends Chat Completions inference
through OpenRouter.
@@ -107,7 +107,7 @@ normalized request shape and routing policy:
```python
import asyncio
-from switchyard.libsy import LlmTarget, algorithms
+from switchyard.libsy import Step, algorithms
from switchyard_litellm import LiteLLMSyClient
@@ -150,22 +150,30 @@ async def main() -> None:
"reasoning": {"effort": "low"},
"output": {"max_output_tokens": 64},
}
- strong = LiteLLMSyClient("strong")
- fast = LiteLLMSyClient("fast")
+ client = LiteLLMSyClient()
router = algorithms.stage_router(
- LlmTarget("strong", strong),
- LlmTarget("fast", fast),
+ "strong",
+ "fast",
picker="efficient_first",
confidence_threshold=0.5,
recent_window=3,
)
try:
- decisions, response = await router.run(request)
- print(decisions)
- print(response)
+ async for step in router.run_stream(request):
+ match step:
+ case Step.Decision(decision):
+ print("Decision:", decision.selected_model_id, decision.reasoning)
+ case Step.CallModel(call):
+ try:
+ response = await client.call(call.request)
+ except Exception as error:
+ call.fail(error)
+ else:
+ call.respond(response)
+ case Step.Done(response):
+ print("Response:", response)
finally:
- await strong.aclose()
- await fast.aclose()
+ await client.aclose()
asyncio.run(main())
diff --git a/examples/experimental/litellm/example.py b/examples/experimental/litellm/example.py
index bb67c27ea..df2691865 100644
--- a/examples/experimental/litellm/example.py
+++ b/examples/experimental/litellm/example.py
@@ -8,7 +8,7 @@ import asyncio
from switchyard_litellm import LiteLLMSyClient
-from switchyard.libsy import LlmTarget, algorithms
+from switchyard.libsy import Step, algorithms
def sy_request() -> dict[str, object]:
@@ -60,21 +60,30 @@ def sy_request() -> dict[str, object]:
async def main() -> None:
"""Run the Stage router and print its normalized result."""
- strong_client = LiteLLMSyClient("strong")
- fast_client = LiteLLMSyClient("fast")
+ client = LiteLLMSyClient()
router = algorithms.stage_router(
- LlmTarget("strong", strong_client),
- LlmTarget("fast", fast_client),
+ "strong",
+ "fast",
picker="efficient_first",
confidence_threshold=0.5,
recent_window=3,
)
try:
- decisions, response = await router.run(sy_request())
- print("Stage router:", decisions, response)
+ async for step in router.run_stream(sy_request()):
+ match step:
+ case Step.Decision(decision):
+ print("Decision:", decision.selected_model_id, decision.reasoning)
+ case Step.CallModel(call):
+ try:
+ response = await client.call(call.request)
+ except Exception as error:
+ call.fail(error)
+ else:
+ call.respond(response)
+ case Step.Done(response):
+ print("Response:", response)
finally:
- await strong_client.aclose()
- await fast_client.aclose()
+ await client.aclose()
if __name__ == "__main__":
diff --git a/examples/experimental/litellm/src/switchyard_litellm/client.py b/examples/experimental/litellm/src/switchyard_litellm/client.py
index dc0286c1f..dd83b5b0a 100644
--- a/examples/experimental/litellm/src/switchyard_litellm/client.py
+++ b/examples/experimental/litellm/src/switchyard_litellm/client.py
@@ -10,6 +10,11 @@ from collections.abc import Mapping, Sequence
from typing import Any, cast
from litellm import ModelResponse, acompletion
+from litellm.exceptions import (
+ ContextWindowExceededError as LiteLLMContextWindowExceededError,
+)
+
+from switchyard.libsy import ContextWindowExceededError
_TEXT_ROLES = {"system", "developer", "user"}
_STOP_REASONS = {
@@ -208,7 +213,10 @@ def _optional_mapping(
return _mapping(value, field)
-def _payload(request: Mapping[str, object], model: str) -> dict[str, Any]:
+async def _payload(request: Mapping[str, object]) -> dict[str, Any]:
+ model = request.get("model")
+ if not isinstance(model, str) or not model:
+ raise ValueError("model must be a non-empty string")
if request.get("stream") is True:
raise ValueError("stream=True is not supported")
_reject_sequence_payload(request, "instructions")
@@ -226,7 +234,9 @@ def _payload(request: Mapping[str, object], model: str) -> dict[str, Any]:
raise ValueError("reasoning.raw is not supported")
payload: dict[str, Any] = {
- "model": model,
+ # LiteLLM needs this prefix to select its OpenAI-compatible transport,
+ # then strips it before sending the selected alias to the gateway.
+ "model": f"openai/{model}",
"messages": _messages(request),
"stream": False,
}
@@ -307,18 +317,14 @@ def _response(response: ModelResponse) -> dict[str, object]:
class LiteLLMSyClient:
- """Call a LiteLLM gateway alias for a libsy target."""
+ """Call the LiteLLM gateway alias selected in a libsy request."""
def __init__(
self,
- model: str,
*,
base_url: str = "http://127.0.0.1:4000/v1",
api_key: str = "not-needed",
) -> None:
- if not model:
- raise ValueError("model must not be empty")
- self.model = model
self._base_url = base_url
self._api_key = api_key
@@ -327,17 +333,21 @@ class LiteLLMSyClient:
sy_request: Mapping[str, object],
) -> Mapping[str, object]:
"""Send one normalized, buffered text request through LiteLLM."""
- response = await acompletion(
- **_payload(sy_request, f"openai/{self.model}"),
- api_base=self._base_url,
- api_key=self._api_key,
- num_retries=0,
- # LiteLLM otherwise imports its optional proxy MCP stack for ordinary
- # OpenAI function tools before it determines that they are not MCP tools.
- _skip_mcp_handler=True,
- allowed_openai_params=["reasoning_effort"],
- extra_body={"allowed_openai_params": ["reasoning_effort"]},
- )
+ payload = await _payload(sy_request)
+ try:
+ response = await acompletion(
+ **payload,
+ api_base=self._base_url,
+ api_key=self._api_key,
+ num_retries=0,
+ # LiteLLM otherwise imports its optional proxy MCP stack for ordinary
+ # OpenAI function tools before it determines that they are not MCP tools.
+ _skip_mcp_handler=True,
+ allowed_openai_params=["reasoning_effort"],
+ extra_body={"allowed_openai_params": ["reasoning_effort"]},
+ )
+ except LiteLLMContextWindowExceededError as error:
+ raise ContextWindowExceededError(str(error)) from error
return _response(cast(ModelResponse, response))
async def aclose(self) -> None:
diff --git a/examples/experimental/litellm/tests/test_client.py b/examples/experimental/litellm/tests/test_client.py
index 5e3654816..157b35c05 100644
--- a/examples/experimental/litellm/tests/test_client.py
+++ b/examples/experimental/litellm/tests/test_client.py
@@ -7,15 +7,24 @@ import json
import httpx
import pytest
import respx
-from litellm import AuthenticationError, ModelResponse, RateLimitError
+from litellm import (
+ AuthenticationError,
+ ModelResponse,
+ RateLimitError,
+)
+from litellm.exceptions import (
+ ContextWindowExceededError as LiteLLMContextWindowExceededError,
+)
from switchyard_litellm import LiteLLMSyClient
+from switchyard.libsy import ContextWindowExceededError
+
BASE_URL = "http://gateway.test/v1"
def request_body() -> dict[str, object]:
return {
- "model": "auto",
+ "model": "fast",
"instructions": [],
"messages": [
{
@@ -87,7 +96,7 @@ async def test_call_uses_litellm_async_completion(
return ModelResponse(**gateway_response())
monkeypatch.setattr("switchyard_litellm.client.acompletion", fake_acompletion)
- client = LiteLLMSyClient("fast", base_url=BASE_URL)
+ client = LiteLLMSyClient(base_url=BASE_URL)
try:
await client.call(request_body())
finally:
@@ -108,7 +117,7 @@ async def test_call_translates_request_and_normalizes_response() -> None:
)
request = request_body()
original = copy.deepcopy(request)
- client = LiteLLMSyClient("fast", base_url=BASE_URL)
+ client = LiteLLMSyClient(base_url=BASE_URL)
try:
response = await client.call(request)
finally:
@@ -200,7 +209,7 @@ async def test_call_translates_function_tool_history() -> None:
]
request["tool_choice"] = {"type": "tool", "data": {"name": "Bash"}}
- client = LiteLLMSyClient("fast", base_url=BASE_URL)
+ client = LiteLLMSyClient(base_url=BASE_URL)
try:
await client.call(request)
finally:
@@ -255,7 +264,7 @@ async def test_call_normalizes_a_function_tool_response(
return ModelResponse(**gateway_tool_response())
monkeypatch.setattr("switchyard_litellm.client.acompletion", fake_acompletion)
- client = LiteLLMSyClient("fast", base_url=BASE_URL)
+ client = LiteLLMSyClient(base_url=BASE_URL)
try:
response = await client.call(request_body())
finally:
@@ -298,7 +307,7 @@ async def test_call_rejects_non_mapping_tool_call_arguments(
],
}
]
- client = LiteLLMSyClient("fast", base_url=BASE_URL)
+ client = LiteLLMSyClient(base_url=BASE_URL)
try:
with pytest.raises(ValueError, match=r"messages\[0\]\.content\[0\]\.arguments"):
await client.call(request)
@@ -317,7 +326,7 @@ async def test_call_rejects_non_mapping_response_tool_call_arguments(
return ModelResponse(**payload)
monkeypatch.setattr("switchyard_litellm.client.acompletion", fake_acompletion)
- client = LiteLLMSyClient("fast", base_url=BASE_URL)
+ client = LiteLLMSyClient(base_url=BASE_URL)
try:
with pytest.raises(ValueError, match="invalid tool-call arguments"):
await client.call(request_body())
@@ -340,7 +349,7 @@ async def test_call_translates_standard_tool_choice(
request = request_body()
request["tools"] = [{"name": "Bash", "description": None, "parameters": {}}]
request["tool_choice"] = {"type": choice}
- client = LiteLLMSyClient("fast", base_url=BASE_URL)
+ client = LiteLLMSyClient(base_url=BASE_URL)
try:
await client.call(request)
finally:
@@ -442,7 +451,7 @@ async def test_call_rejects_unsupported_normalized_fields(
router.post(f"{BASE_URL}/chat/completions").mock(
return_value=httpx.Response(200, json=gateway_response())
)
- client = LiteLLMSyClient("fast", base_url=BASE_URL)
+ client = LiteLLMSyClient(base_url=BASE_URL)
try:
with pytest.raises(ValueError, match=match):
await client.call(request)
@@ -451,10 +460,10 @@ async def test_call_rejects_unsupported_normalized_fields(
async def test_call_rejects_missing_messages() -> None:
- client = LiteLLMSyClient("fast", base_url=BASE_URL)
+ client = LiteLLMSyClient(base_url=BASE_URL)
try:
with pytest.raises(ValueError, match="messages"):
- await client.call({})
+ await client.call({"model": "fast"})
finally:
await client.aclose()
@@ -466,7 +475,7 @@ async def test_call_rejects_a_response_without_text() -> None:
respx.post(f"{BASE_URL}/chat/completions").mock(
return_value=httpx.Response(200, json=payload)
)
- client = LiteLLMSyClient("fast", base_url=BASE_URL)
+ client = LiteLLMSyClient(base_url=BASE_URL)
try:
with pytest.raises(ValueError, match="no text content"):
await client.call(request_body())
@@ -484,7 +493,7 @@ async def test_call_rejects_a_response_without_choices(
return ModelResponse(**payload)
monkeypatch.setattr("switchyard_litellm.client.acompletion", fake_acompletion)
- client = LiteLLMSyClient("fast", base_url=BASE_URL)
+ client = LiteLLMSyClient(base_url=BASE_URL)
try:
with pytest.raises(ValueError, match="no choices"):
await client.call(request_body())
@@ -506,7 +515,7 @@ async def test_litellm_errors_propagate() -> None:
},
)
)
- client = LiteLLMSyClient("fast", base_url=BASE_URL)
+ client = LiteLLMSyClient(base_url=BASE_URL)
try:
with pytest.raises(AuthenticationError):
await client.call(request_body())
@@ -528,7 +537,7 @@ async def test_retryable_litellm_error_is_not_retried() -> None:
},
)
)
- client = LiteLLMSyClient("fast", base_url=BASE_URL)
+ client = LiteLLMSyClient(base_url=BASE_URL)
try:
with pytest.raises(RateLimitError):
await client.call(request_body())
@@ -538,6 +547,27 @@ async def test_retryable_litellm_error_is_not_retried() -> None:
assert route.call_count == 1
+async def test_context_window_error_is_preserved(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_with_context_window(**_: object) -> ModelResponse:
+ raise LiteLLMContextWindowExceededError(
+ message="request is too large",
+ model="fast",
+ llm_provider="openai",
+ )
+
+ monkeypatch.setattr(
+ "switchyard_litellm.client.acompletion", fail_with_context_window
+ )
+ client = LiteLLMSyClient(base_url=BASE_URL)
+ try:
+ with pytest.raises(ContextWindowExceededError, match="request is too large"):
+ await client.call(request_body())
+ finally:
+ await client.aclose()
+
+
@respx.mock
async def test_cached_token_count_preserves_explicit_zero() -> None:
payload = gateway_response()
@@ -545,7 +575,7 @@ async def test_cached_token_count_preserves_explicit_zero() -> None:
respx.post(f"{BASE_URL}/chat/completions").mock(
return_value=httpx.Response(200, json=payload)
)
- client = LiteLLMSyClient("fast", base_url=BASE_URL)
+ client = LiteLLMSyClient(base_url=BASE_URL)
try:
response = await client.call(request_body())
finally:
diff --git a/examples/experimental/litellm/tests/test_e2e.py b/examples/experimental/litellm/tests/test_e2e.py
index d4a733357..ad5504681 100644
--- a/examples/experimental/litellm/tests/test_e2e.py
+++ b/examples/experimental/litellm/tests/test_e2e.py
@@ -11,7 +11,7 @@ from pathlib import Path
import pytest
from switchyard_litellm import LiteLLMSyClient
-from switchyard.libsy import LlmTarget, algorithms
+from switchyard.libsy import Algorithm, Decision, Step, algorithms
PACKAGE_ROOT = Path(__file__).resolve().parents[1]
@@ -143,28 +143,45 @@ def _request(*, critical_error: bool = False) -> dict[str, object]:
}
+async def _run_router(
+ router: Algorithm,
+ request: dict[str, object],
+ client: LiteLLMSyClient,
+) -> tuple[list[Decision], dict[str, object]]:
+ decisions: list[Decision] = []
+ async for step in router.run_stream(request):
+ match step:
+ case Step.Decision(decision):
+ decisions.append(decision)
+ case Step.CallModel(call):
+ call.respond(await client.call(call.request))
+ case Step.Done(response):
+ return decisions, response
+ raise AssertionError("algorithm stream ended without a response")
+
+
@pytest.mark.e2e
async def test_stage_router_calls_both_real_openrouter_models(
litellm_base_url: str,
) -> None:
- strong_client = LiteLLMSyClient("strong", base_url=litellm_base_url)
- fast_client = LiteLLMSyClient("fast", base_url=litellm_base_url)
+ client = LiteLLMSyClient(base_url=litellm_base_url)
router = algorithms.stage_router(
- LlmTarget("strong", strong_client),
- LlmTarget("fast", fast_client),
+ "strong",
+ "fast",
picker="efficient_first",
confidence_threshold=0.5,
recent_window=3,
)
try:
- fast_trace, fast_response = await router.run(_request())
- strong_trace, strong_response = await router.run(_request(critical_error=True))
+ fast_trace, fast_response = await _run_router(router, _request(), client)
+ strong_trace, strong_response = await _run_router(
+ router, _request(critical_error=True), client
+ )
finally:
- await strong_client.aclose()
- await fast_client.aclose()
+ await client.aclose()
- assert [item["selected_model"] for item in strong_trace] == ["strong"]
- assert [item["selected_model"] for item in fast_trace] == ["fast"]
+ assert [item.selected_model_id for item in strong_trace] == ["strong"]
+ assert [item.selected_model_id for item in fast_trace] == ["fast"]
for response in (strong_response, fast_response):
text = response["outputs"][0]["content"][0]["text"]
assert isinstance(text, str)
diff --git a/examples/experimental/litellm/tests/test_stage_routing.py b/examples/experimental/litellm/tests/test_stage_routing.py
index 1e8bf4202..54a1e051c 100644
--- a/examples/experimental/litellm/tests/test_stage_routing.py
+++ b/examples/experimental/litellm/tests/test_stage_routing.py
@@ -7,7 +7,7 @@ import httpx
import respx
from switchyard_litellm import LiteLLMSyClient
-from switchyard.libsy import LlmTarget, algorithms
+from switchyard.libsy import Algorithm, Decision, Step, algorithms
BASE_URL = "http://gateway.test/v1"
@@ -73,8 +73,25 @@ def gateway_response(model: str) -> dict[str, object]:
}
+async def _run_router(
+ router: Algorithm,
+ request: dict[str, object],
+ client: LiteLLMSyClient,
+) -> tuple[list[Decision], dict[str, object]]:
+ decisions: list[Decision] = []
+ async for step in router.run_stream(request):
+ match step:
+ case Step.Decision(decision):
+ decisions.append(decision)
+ case Step.CallModel(call):
+ call.respond(await client.call(call.request))
+ case Step.Done(response):
+ return decisions, response
+ raise AssertionError("algorithm stream ended without a response")
+
+
@respx.mock
-async def test_stage_router_drives_both_litellm_targets() -> None:
+async def test_stage_router_drives_both_litellm_models() -> None:
seen: list[str] = []
def respond(request: httpx.Request) -> httpx.Response:
@@ -83,26 +100,26 @@ async def test_stage_router_drives_both_litellm_targets() -> None:
return httpx.Response(200, json=gateway_response(model))
respx.post(f"{BASE_URL}/chat/completions").mock(side_effect=respond)
- strong_client = LiteLLMSyClient("strong", base_url=BASE_URL)
- fast_client = LiteLLMSyClient("fast", base_url=BASE_URL)
+ client = LiteLLMSyClient(base_url=BASE_URL)
router = algorithms.stage_router(
- LlmTarget("strong", strong_client),
- LlmTarget("fast", fast_client),
+ "strong",
+ "fast",
picker="efficient_first",
confidence_threshold=0.5,
recent_window=3,
)
try:
- fast_decisions, fast_response = await router.run(request_body())
- strong_decisions, strong_response = await router.run(
- request_body(critical_error=True)
+ fast_decisions, fast_response = await _run_router(
+ router, request_body(), client
+ )
+ strong_decisions, strong_response = await _run_router(
+ router, request_body(critical_error=True), client
)
finally:
- await strong_client.aclose()
- await fast_client.aclose()
+ await client.aclose()
- assert [item["selected_model"] for item in fast_decisions] == ["fast"]
- assert [item["selected_model"] for item in strong_decisions] == ["strong"]
+ assert [item.selected_model_id for item in fast_decisions] == ["fast"]
+ assert [item.selected_model_id for item in strong_decisions] == ["strong"]
assert fast_response["outputs"][0]["content"][0]["text"] == "fast"
assert strong_response["outputs"][0]["content"][0]["text"] == "strong"
assert seen == ["fast", "strong"]
diff --git a/examples/experimental/litellm/uv.lock b/examples/experimental/litellm/uv.lock
index b7b6e9702..656b5b6a8 100644
--- a/examples/experimental/litellm/uv.lock
+++ b/examples/experimental/litellm/uv.lock
@@ -69,25 +69,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" },
]
-[[package]]
-name = "anthropic"
-version = "0.120.2"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
- { name = "anyio" },
- { name = "distro" },
- { name = "docstring-parser" },
- { name = "httpx" },
- { name = "jiter" },
- { name = "pydantic" },
- { name = "sniffio" },
- { name = "typing-extensions" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/d7/10/4ca013cb166f226bd89e0aeb0fcaff94f45ddf716d4925ce89475d3c587b/anthropic-0.120.2.tar.gz", hash = "sha256:9722efc10c27a30a69f5338ddacdb35bc6a64297a4e4ba729bf83af873d5fb3a", size = 1008421, upload-time = "2026-07-28T17:38:26.986Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/63/af/0f5db57b9397a0f3b7fc204cbef143401a7cadaf982330f97f1ce3d39f34/anthropic-0.120.2-py3-none-any.whl", hash = "sha256:0f0bc2b381dc0eb41c8d886b815d79c2041cd2374f83aed36f574b6dc9c579c1", size = 1022851, upload-time = "2026-07-28T17:38:25.466Z" },
-]
-
[[package]]
name = "anyio"
version = "4.14.2"
@@ -171,15 +152,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" },
]
-[[package]]
-name = "docstring-parser"
-version = "0.18.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" },
-]
-
[[package]]
name = "fastuuid"
version = "0.14.0"
@@ -484,46 +456,24 @@ wheels = [
name = "nemo-switchyard"
version = "0.2.0"
source = { editable = "../../../" }
-dependencies = [
- { name = "anthropic" },
- { name = "httpx" },
- { name = "openai" },
- { name = "pydantic" },
-]
[package.metadata]
-requires-dist = [
- { name = "anthropic", specifier = ">=0.99.0,<1.0" },
- { name = "ddtrace", marker = "extra == 'tracing'", specifier = ">=2.9,<4" },
- { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.136.1,<1.0" },
- { name = "httpx", specifier = ">=0.28.1,<1.0" },
- { name = "nemo-switchyard", extras = ["server", "cli", "tracing", "affinity-redis"], marker = "extra == 'all'" },
- { name = "openai", specifier = ">=2.7,<3.0" },
- { name = "prompt-toolkit", marker = "extra == 'cli'", specifier = ">=3.0.52,<4.0" },
- { name = "pydantic", specifier = ">=2.13.3,<3.0" },
- { name = "redis", marker = "extra == 'affinity-redis'", specifier = ">=5.0.1,<6" },
- { name = "sse-starlette", marker = "extra == 'server'", specifier = ">=3.4.1,<4.0" },
- { name = "uvicorn", extras = ["standard"], marker = "extra == 'server'", specifier = ">=0.46.0,<1.0" },
-]
-provides-extras = ["server", "cli", "tracing", "affinity-redis", "all"]
+requires-dist = [{ name = "prompt-toolkit", marker = "extra == 'cli'", specifier = ">=3.0.52,<4.0" }]
+provides-extras = ["cli"]
[package.metadata.requires-dev]
dev = [
{ name = "harbor", marker = "python_full_version >= '3.12'", git = "https://github.com/harbor-framework/harbor.git?rev=v0.6.4" },
- { name = "httpx", specifier = ">=0.28.1,<1.0" },
{ name = "maturin", specifier = ">=1.9,<2.0" },
{ name = "mypy", specifier = ">=1.20.2,<2.0" },
- { name = "nemo-switchyard", extras = ["server"] },
- { name = "prometheus-client", specifier = ">=0.21.0,<1.0" },
+ { name = "nemo-switchyard", extras = ["cli"] },
{ name = "pytest", specifier = ">=9.0.3,<10.0" },
{ name = "pytest-asyncio", specifier = ">=1.3.0,<2.0" },
- { name = "pytest-cov", specifier = ">=7.1.0,<8.0" },
{ name = "pytest-markdown-docs", specifier = ">=0.9.2" },
- { name = "pytest-mock", specifier = ">=3.15.1,<4.0" },
{ name = "pytest-timeout", specifier = ">=2.4.0,<3.0" },
- { name = "respx", specifier = ">=0.23.1,<1.0" },
+ { name = "pyyaml", specifier = ">=6.0.3,<7.0" },
{ name = "ruff", specifier = ">=0.15.12,<1.0" },
- { name = "socksio", specifier = ">=1.0.0,<2.0" },
+ { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0,<3.0" },
]
docs = [
{ name = "mkdocs", specifier = ">=1.6.0,<2.0" },
diff --git a/switchyard/libsy/__init__.py b/switchyard/libsy/__init__.py
index 6d2b5ba74..78a535d01 100644
--- a/switchyard/libsy/__init__.py
+++ b/switchyard/libsy/__init__.py
@@ -5,6 +5,7 @@
from switchyard_rust.libsy import (
Algorithm,
+ ContextWindowExceededError,
Decision,
LibsyError,
LlmFallback,
@@ -17,6 +18,7 @@ from . import algorithms as algorithms
__all__ = [
"Algorithm",
+ "ContextWindowExceededError",
"Decision",
"LibsyError",
"LlmFallback",
diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py
index 244b1dd02..5dd607566 100644
--- a/switchyard_rust/libsy.py
+++ b/switchyard_rust/libsy.py
@@ -13,6 +13,7 @@ from switchyard_rust._native import load_native
_EXPORTS = frozenset(
{
"Algorithm",
+ "ContextWindowExceededError",
"Decision",
"LibsyError",
"LlmFallback",
@@ -32,6 +33,8 @@ if TYPE_CHECKING:
class LibsyError(RuntimeError): ...
+ class ContextWindowExceededError(RuntimeError): ...
+
@final
class Decision:
"""A semantic routing choice produced by an algorithm."""
diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py
index eb1d6c8ce..988454635 100644
--- a/tests/test_libsy_minimal_bindings.py
+++ b/tests/test_libsy_minimal_bindings.py
@@ -9,6 +9,7 @@ import pytest
from switchyard.libsy import (
Algorithm,
+ ContextWindowExceededError,
Decision,
LibsyError,
Step,
@@ -273,3 +274,23 @@ async def test_client_failure_becomes_libsy_error() -> None:
with pytest.raises(LibsyError, match="client failed"):
await run_algorithm(algorithm, {"broken": FailingClient()})
+
+
+async def test_context_window_failure_falls_back_to_the_next_model() -> None:
+ class OverflowClient:
+ async def call(self, request: dict[str, Any]) -> dict[str, Any]:
+ raise ContextWindowExceededError("request exceeds context window")
+
+ algorithm = algorithms.stage_router(
+ "strong",
+ "fast",
+ picker="efficient_first",
+ confidence_threshold=0.5,
+ )
+ decisions, response = await run_algorithm(
+ algorithm,
+ {"fast": OverflowClient(), "strong": EchoClient("strong")},
+ )
+
+ assert [decision.selected_model_id for decision in decisions] == ["fast", "strong"]
+ assert response["model"] == "strong"