Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(examples): update LiteLLM for run stream (#412)
nachiketb-nvidiae9767d642461
diff --git a/crates/switchyard-py/src/errors.rs b/crates/switchyard-py/src/errors.rsindex 9e0beb775..8f5e44e47 100644--- a/crates/switchyard-py/src/errors.rs+++ b/crates/switchyard-py/src/errors.rs@@ -8,6 +8,7 @@ use pyo3::exceptions::PyRuntimeError;use pyo3::prelude::*;create_exception!(_switchyard_rust, LibsyError, PyRuntimeError);+create_exception!(_switchyard_rust, ContextWindowExceededError, PyRuntimeError);/// Converts libsy execution failures into one stable Python exception.pub(crate) fn py_libsy_error(error: impl std::fmt::Display) -> PyErr {@@ -15,5 +16,9 @@ pub(crate) fn py_libsy_error(error: impl std::fmt::Display) -> PyErr {}pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {- module.add("LibsyError", module.py().get_type::<LibsyError>())+ module.add("LibsyError", module.py().get_type::<LibsyError>())?;+ module.add(+ "ContextWindowExceededError",+ module.py().get_type::<ContextWindowExceededError>(),+ )}diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rsindex ee2b70fd9..ce303d5e6 100644--- a/crates/switchyard-py/src/libsy_bindings.rs+++ b/crates/switchyard-py/src/libsy_bindings.rs@@ -21,7 +21,7 @@ use switchyard_protocol::{};use tokio::sync::Mutex;-use crate::errors::py_libsy_error;+use crate::errors::{ContextWindowExceededError, py_libsy_error};use crate::py_serde::{from_python, to_python};/// Convert Python-owned headers into the request metadata expected by libsy.@@ -250,8 +250,15 @@ impl PyModelCall {}let call = self.take()?;let target = call.decision.selected_model_id().clone();- let source = LlmClientError::Ffi {- source: Box::new(PyErr::from_value(error.clone())),+ let source = if error.is_instance_of::<ContextWindowExceededError>() {+ LlmClientError::ContextWindowExceeded {+ model: target.clone(),+ message: error.str()?.to_string_lossy().into_owned(),+ }+ } else {+ LlmClientError::Ffi {+ source: Box::new(PyErr::from_value(error.clone())),+ }};call.respond(Err(RustLibsyError::client_call(target, source))).map_err(py_libsy_error)@@ -505,6 +512,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {&libsy_module)?)?;libsy_module.add_function(wrap_pyfunction!(stage_router_algorithm, &libsy_module)?)?;+ libsy_module.add(+ "ContextWindowExceededError",+ module.getattr("ContextWindowExceededError")?,+ )?;libsy_module.add("LibsyError", module.getattr("LibsyError")?)?;module.add_submodule(&libsy_module)?;Ok(())diff --git a/examples/experimental/litellm/README.md b/examples/experimental/litellm/README.mdindex 1f2294e9a..836cca915 100644--- a/examples/experimental/litellm/README.md+++ b/examples/experimental/litellm/README.md@@ -11,8 +11,8 @@ point for your own application.LiteLLM provides the OpenAI-compatible gateway, model aliases, and OpenRouterprovider integration. Switchyard's Stage router makes the routing decision fromthe coding agent's recent tool history.-`LiteLLMSyClient` connects Switchyard's normalized libsy requests to the-selected alias through LiteLLM's asynchronous Completion API and the+`LiteLLMSyClient` connects Switchyard's normalized libsy requests to their+selected aliases through LiteLLM's asynchronous Completion API and theDockerized gateway. Together, they let an application keep routing policy inSwitchyard while LiteLLM owns model access and sends Chat Completions inferencethrough OpenRouter.@@ -107,7 +107,7 @@ normalized request shape and routing policy:```pythonimport asyncio-from switchyard.libsy import LlmTarget, algorithms+from switchyard.libsy import Step, algorithmsfrom switchyard_litellm import LiteLLMSyClient@@ -150,22 +150,30 @@ async def main() -> None:"reasoning": {"effort": "low"},"output": {"max_output_tokens": 64},}- strong = LiteLLMSyClient("strong")- fast = LiteLLMSyClient("fast")+ client = LiteLLMSyClient()router = algorithms.stage_router(- LlmTarget("strong", strong),- LlmTarget("fast", fast),+ "strong",+ "fast",picker="efficient_first",confidence_threshold=0.5,recent_window=3,)try:- decisions, response = await router.run(request)- print(decisions)- print(response)+ async for step in router.run_stream(request):+ match step:+ case Step.Decision(decision):+ print("Decision:", decision.selected_model_id, decision.reasoning)+ case Step.CallModel(call):+ try:+ response = await client.call(call.request)+ except Exception as error:+ call.fail(error)+ else:+ call.respond(response)+ case Step.Done(response):+ print("Response:", response)finally:- await strong.aclose()- await fast.aclose()+ await client.aclose()asyncio.run(main())diff --git a/examples/experimental/litellm/example.py b/examples/experimental/litellm/example.pyindex bb67c27ea..df2691865 100644--- a/examples/experimental/litellm/example.py+++ b/examples/experimental/litellm/example.py@@ -8,7 +8,7 @@ import asynciofrom switchyard_litellm import LiteLLMSyClient-from switchyard.libsy import LlmTarget, algorithms+from switchyard.libsy import Step, algorithmsdef sy_request() -> dict[str, object]:@@ -60,21 +60,30 @@ def sy_request() -> dict[str, object]:async def main() -> None:"""Run the Stage router and print its normalized result."""- strong_client = LiteLLMSyClient("strong")- fast_client = LiteLLMSyClient("fast")+ client = LiteLLMSyClient()router = algorithms.stage_router(- LlmTarget("strong", strong_client),- LlmTarget("fast", fast_client),+ "strong",+ "fast",picker="efficient_first",confidence_threshold=0.5,recent_window=3,)try:- decisions, response = await router.run(sy_request())- print("Stage router:", decisions, response)+ async for step in router.run_stream(sy_request()):+ match step:+ case Step.Decision(decision):+ print("Decision:", decision.selected_model_id, decision.reasoning)+ case Step.CallModel(call):+ try:+ response = await client.call(call.request)+ except Exception as error:+ call.fail(error)+ else:+ call.respond(response)+ case Step.Done(response):+ print("Response:", response)finally:- await strong_client.aclose()- await fast_client.aclose()+ await client.aclose()if __name__ == "__main__":diff --git a/examples/experimental/litellm/src/switchyard_litellm/client.py b/examples/experimental/litellm/src/switchyard_litellm/client.pyindex dc0286c1f..dd83b5b0a 100644--- a/examples/experimental/litellm/src/switchyard_litellm/client.py+++ b/examples/experimental/litellm/src/switchyard_litellm/client.py@@ -10,6 +10,11 @@ from collections.abc import Mapping, Sequencefrom typing import Any, castfrom litellm import ModelResponse, acompletion+from litellm.exceptions import (+ ContextWindowExceededError as LiteLLMContextWindowExceededError,+)++from switchyard.libsy import ContextWindowExceededError_TEXT_ROLES = {"system", "developer", "user"}_STOP_REASONS = {@@ -208,7 +213,10 @@ def _optional_mapping(return _mapping(value, field)-def _payload(request: Mapping[str, object], model: str) -> dict[str, Any]:+async def _payload(request: Mapping[str, object]) -> dict[str, Any]:+ model = request.get("model")+ if not isinstance(model, str) or not model:+ raise ValueError("model must be a non-empty string")if request.get("stream") is True:raise ValueError("stream=True is not supported")_reject_sequence_payload(request, "instructions")@@ -226,7 +234,9 @@ def _payload(request: Mapping[str, object], model: str) -> dict[str, Any]:raise ValueError("reasoning.raw is not supported")payload: dict[str, Any] = {- "model": model,+ # LiteLLM needs this prefix to select its OpenAI-compatible transport,+ # then strips it before sending the selected alias to the gateway.+ "model": f"openai/{model}","messages": _messages(request),"stream": False,}@@ -307,18 +317,14 @@ def _response(response: ModelResponse) -> dict[str, object]:class LiteLLMSyClient:- """Call a LiteLLM gateway alias for a libsy target."""+ """Call the LiteLLM gateway alias selected in a libsy request."""def __init__(self,- model: str,*,base_url: str = "http://127.0.0.1:4000/v1",api_key: str = "not-needed",) -> None:- if not model:- raise ValueError("model must not be empty")- self.model = modelself._base_url = base_urlself._api_key = api_key@@ -327,17 +333,21 @@ class LiteLLMSyClient:sy_request: Mapping[str, object],) -> Mapping[str, object]:"""Send one normalized, buffered text request through LiteLLM."""- response = await acompletion(- **_payload(sy_request, f"openai/{self.model}"),- api_base=self._base_url,- api_key=self._api_key,- num_retries=0,- # LiteLLM otherwise imports its optional proxy MCP stack for ordinary- # OpenAI function tools before it determines that they are not MCP tools.- _skip_mcp_handler=True,- allowed_openai_params=["reasoning_effort"],- extra_body={"allowed_openai_params": ["reasoning_effort"]},- )+ payload = await _payload(sy_request)+ try:+ response = await acompletion(+ **payload,+ api_base=self._base_url,+ api_key=self._api_key,+ num_retries=0,+ # LiteLLM otherwise imports its optional proxy MCP stack for ordinary+ # OpenAI function tools before it determines that they are not MCP tools.+ _skip_mcp_handler=True,+ allowed_openai_params=["reasoning_effort"],+ extra_body={"allowed_openai_params": ["reasoning_effort"]},+ )+ except LiteLLMContextWindowExceededError as error:+ raise ContextWindowExceededError(str(error)) from errorreturn _response(cast(ModelResponse, response))async def aclose(self) -> None:diff --git a/examples/experimental/litellm/tests/test_client.py b/examples/experimental/litellm/tests/test_client.pyindex 5e3654816..157b35c05 100644--- a/examples/experimental/litellm/tests/test_client.py+++ b/examples/experimental/litellm/tests/test_client.py@@ -7,15 +7,24 @@ import jsonimport httpximport pytestimport respx-from litellm import AuthenticationError, ModelResponse, RateLimitError+from litellm import (+ AuthenticationError,+ ModelResponse,+ RateLimitError,+)+from litellm.exceptions import (+ ContextWindowExceededError as LiteLLMContextWindowExceededError,+)from switchyard_litellm import LiteLLMSyClient+from switchyard.libsy import ContextWindowExceededError+BASE_URL = "http://gateway.test/v1"def request_body() -> dict[str, object]:return {- "model": "auto",+ "model": "fast","instructions": [],"messages": [{@@ -87,7 +96,7 @@ async def test_call_uses_litellm_async_completion(return ModelResponse(**gateway_response())monkeypatch.setattr("switchyard_litellm.client.acompletion", fake_acompletion)- client = LiteLLMSyClient("fast", base_url=BASE_URL)+ client = LiteLLMSyClient(base_url=BASE_URL)try:await client.call(request_body())finally:@@ -108,7 +117,7 @@ async def test_call_translates_request_and_normalizes_response() -> None:)request = request_body()original = copy.deepcopy(request)- client = LiteLLMSyClient("fast", base_url=BASE_URL)+ client = LiteLLMSyClient(base_url=BASE_URL)try:response = await client.call(request)finally:@@ -200,7 +209,7 @@ async def test_call_translates_function_tool_history() -> None:]request["tool_choice"] = {"type": "tool", "data": {"name": "Bash"}}- client = LiteLLMSyClient("fast", base_url=BASE_URL)+ client = LiteLLMSyClient(base_url=BASE_URL)try:await client.call(request)finally:@@ -255,7 +264,7 @@ async def test_call_normalizes_a_function_tool_response(return ModelResponse(**gateway_tool_response())monkeypatch.setattr("switchyard_litellm.client.acompletion", fake_acompletion)- client = LiteLLMSyClient("fast", base_url=BASE_URL)+ client = LiteLLMSyClient(base_url=BASE_URL)try:response = await client.call(request_body())finally:@@ -298,7 +307,7 @@ async def test_call_rejects_non_mapping_tool_call_arguments(],}]- client = LiteLLMSyClient("fast", base_url=BASE_URL)+ client = LiteLLMSyClient(base_url=BASE_URL)try:with pytest.raises(ValueError, match=r"messages\[0\]\.content\[0\]\.arguments"):await client.call(request)@@ -317,7 +326,7 @@ async def test_call_rejects_non_mapping_response_tool_call_arguments(return ModelResponse(**payload)monkeypatch.setattr("switchyard_litellm.client.acompletion", fake_acompletion)- client = LiteLLMSyClient("fast", base_url=BASE_URL)+ client = LiteLLMSyClient(base_url=BASE_URL)try:with pytest.raises(ValueError, match="invalid tool-call arguments"):await client.call(request_body())@@ -340,7 +349,7 @@ async def test_call_translates_standard_tool_choice(request = request_body()request["tools"] = [{"name": "Bash", "description": None, "parameters": {}}]request["tool_choice"] = {"type": choice}- client = LiteLLMSyClient("fast", base_url=BASE_URL)+ client = LiteLLMSyClient(base_url=BASE_URL)try:await client.call(request)finally:@@ -442,7 +451,7 @@ async def test_call_rejects_unsupported_normalized_fields(router.post(f"{BASE_URL}/chat/completions").mock(return_value=httpx.Response(200, json=gateway_response()))- client = LiteLLMSyClient("fast", base_url=BASE_URL)+ client = LiteLLMSyClient(base_url=BASE_URL)try:with pytest.raises(ValueError, match=match):await client.call(request)@@ -451,10 +460,10 @@ async def test_call_rejects_unsupported_normalized_fields(async def test_call_rejects_missing_messages() -> None:- client = LiteLLMSyClient("fast", base_url=BASE_URL)+ client = LiteLLMSyClient(base_url=BASE_URL)try:with pytest.raises(ValueError, match="messages"):- await client.call({})+ await client.call({"model": "fast"})finally:await client.aclose()@@ -466,7 +475,7 @@ async def test_call_rejects_a_response_without_text() -> None:respx.post(f"{BASE_URL}/chat/completions").mock(return_value=httpx.Response(200, json=payload))- client = LiteLLMSyClient("fast", base_url=BASE_URL)+ client = LiteLLMSyClient(base_url=BASE_URL)try:with pytest.raises(ValueError, match="no text content"):await client.call(request_body())@@ -484,7 +493,7 @@ async def test_call_rejects_a_response_without_choices(return ModelResponse(**payload)monkeypatch.setattr("switchyard_litellm.client.acompletion", fake_acompletion)- client = LiteLLMSyClient("fast", base_url=BASE_URL)+ client = LiteLLMSyClient(base_url=BASE_URL)try:with pytest.raises(ValueError, match="no choices"):await client.call(request_body())@@ -506,7 +515,7 @@ async def test_litellm_errors_propagate() -> None:},))- client = LiteLLMSyClient("fast", base_url=BASE_URL)+ client = LiteLLMSyClient(base_url=BASE_URL)try:with pytest.raises(AuthenticationError):await client.call(request_body())@@ -528,7 +537,7 @@ async def test_retryable_litellm_error_is_not_retried() -> None:},))- client = LiteLLMSyClient("fast", base_url=BASE_URL)+ client = LiteLLMSyClient(base_url=BASE_URL)try:with pytest.raises(RateLimitError):await client.call(request_body())@@ -538,6 +547,27 @@ async def test_retryable_litellm_error_is_not_retried() -> None:assert route.call_count == 1+async def test_context_window_error_is_preserved(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_with_context_window(**_: object) -> ModelResponse:+ raise LiteLLMContextWindowExceededError(+ message="request is too large",+ model="fast",+ llm_provider="openai",+ )++ monkeypatch.setattr(+ "switchyard_litellm.client.acompletion", fail_with_context_window+ )+ client = LiteLLMSyClient(base_url=BASE_URL)+ try:+ with pytest.raises(ContextWindowExceededError, match="request is too large"):+ await client.call(request_body())+ finally:+ await client.aclose()++@respx.mockasync def test_cached_token_count_preserves_explicit_zero() -> None:payload = gateway_response()@@ -545,7 +575,7 @@ async def test_cached_token_count_preserves_explicit_zero() -> None:respx.post(f"{BASE_URL}/chat/completions").mock(return_value=httpx.Response(200, json=payload))- client = LiteLLMSyClient("fast", base_url=BASE_URL)+ client = LiteLLMSyClient(base_url=BASE_URL)try:response = await client.call(request_body())finally:diff --git a/examples/experimental/litellm/tests/test_e2e.py b/examples/experimental/litellm/tests/test_e2e.pyindex d4a733357..ad5504681 100644--- a/examples/experimental/litellm/tests/test_e2e.py+++ b/examples/experimental/litellm/tests/test_e2e.py@@ -11,7 +11,7 @@ from pathlib import Pathimport pytestfrom switchyard_litellm import LiteLLMSyClient-from switchyard.libsy import LlmTarget, algorithms+from switchyard.libsy import Algorithm, Decision, Step, algorithmsPACKAGE_ROOT = Path(__file__).resolve().parents[1]@@ -143,28 +143,45 @@ def _request(*, critical_error: bool = False) -> dict[str, object]:}+async def _run_router(+ router: Algorithm,+ request: dict[str, object],+ client: LiteLLMSyClient,+) -> tuple[list[Decision], dict[str, object]]:+ decisions: list[Decision] = []+ async for step in router.run_stream(request):+ match step:+ case Step.Decision(decision):+ decisions.append(decision)+ case Step.CallModel(call):+ call.respond(await client.call(call.request))+ case Step.Done(response):+ return decisions, response+ raise AssertionError("algorithm stream ended without a response")++@pytest.mark.e2easync def test_stage_router_calls_both_real_openrouter_models(litellm_base_url: str,) -> None:- strong_client = LiteLLMSyClient("strong", base_url=litellm_base_url)- fast_client = LiteLLMSyClient("fast", base_url=litellm_base_url)+ client = LiteLLMSyClient(base_url=litellm_base_url)router = algorithms.stage_router(- LlmTarget("strong", strong_client),- LlmTarget("fast", fast_client),+ "strong",+ "fast",picker="efficient_first",confidence_threshold=0.5,recent_window=3,)try:- fast_trace, fast_response = await router.run(_request())- strong_trace, strong_response = await router.run(_request(critical_error=True))+ fast_trace, fast_response = await _run_router(router, _request(), client)+ strong_trace, strong_response = await _run_router(+ router, _request(critical_error=True), client+ )finally:- await strong_client.aclose()- await fast_client.aclose()+ await client.aclose()- assert [item["selected_model"] for item in strong_trace] == ["strong"]- assert [item["selected_model"] for item in fast_trace] == ["fast"]+ assert [item.selected_model_id for item in strong_trace] == ["strong"]+ assert [item.selected_model_id for item in fast_trace] == ["fast"]for response in (strong_response, fast_response):text = response["outputs"][0]["content"][0]["text"]assert isinstance(text, str)diff --git a/examples/experimental/litellm/tests/test_stage_routing.py b/examples/experimental/litellm/tests/test_stage_routing.pyindex 1e8bf4202..54a1e051c 100644--- a/examples/experimental/litellm/tests/test_stage_routing.py+++ b/examples/experimental/litellm/tests/test_stage_routing.py@@ -7,7 +7,7 @@ import httpximport respxfrom switchyard_litellm import LiteLLMSyClient-from switchyard.libsy import LlmTarget, algorithms+from switchyard.libsy import Algorithm, Decision, Step, algorithmsBASE_URL = "http://gateway.test/v1"@@ -73,8 +73,25 @@ def gateway_response(model: str) -> dict[str, object]:}+async def _run_router(+ router: Algorithm,+ request: dict[str, object],+ client: LiteLLMSyClient,+) -> tuple[list[Decision], dict[str, object]]:+ decisions: list[Decision] = []+ async for step in router.run_stream(request):+ match step:+ case Step.Decision(decision):+ decisions.append(decision)+ case Step.CallModel(call):+ call.respond(await client.call(call.request))+ case Step.Done(response):+ return decisions, response+ raise AssertionError("algorithm stream ended without a response")++@respx.mock-async def test_stage_router_drives_both_litellm_targets() -> None:+async def test_stage_router_drives_both_litellm_models() -> None:seen: list[str] = []def respond(request: httpx.Request) -> httpx.Response:@@ -83,26 +100,26 @@ async def test_stage_router_drives_both_litellm_targets() -> None:return httpx.Response(200, json=gateway_response(model))respx.post(f"{BASE_URL}/chat/completions").mock(side_effect=respond)- strong_client = LiteLLMSyClient("strong", base_url=BASE_URL)- fast_client = LiteLLMSyClient("fast", base_url=BASE_URL)+ client = LiteLLMSyClient(base_url=BASE_URL)router = algorithms.stage_router(- LlmTarget("strong", strong_client),- LlmTarget("fast", fast_client),+ "strong",+ "fast",picker="efficient_first",confidence_threshold=0.5,recent_window=3,)try:- fast_decisions, fast_response = await router.run(request_body())- strong_decisions, strong_response = await router.run(- request_body(critical_error=True)+ fast_decisions, fast_response = await _run_router(+ router, request_body(), client+ )+ strong_decisions, strong_response = await _run_router(+ router, request_body(critical_error=True), client)finally:- await strong_client.aclose()- await fast_client.aclose()+ await client.aclose()- assert [item["selected_model"] for item in fast_decisions] == ["fast"]- assert [item["selected_model"] for item in strong_decisions] == ["strong"]+ assert [item.selected_model_id for item in fast_decisions] == ["fast"]+ assert [item.selected_model_id for item in strong_decisions] == ["strong"]assert fast_response["outputs"][0]["content"][0]["text"] == "fast"assert strong_response["outputs"][0]["content"][0]["text"] == "strong"assert seen == ["fast", "strong"]diff --git a/examples/experimental/litellm/uv.lock b/examples/experimental/litellm/uv.lockindex b7b6e9702..656b5b6a8 100644--- a/examples/experimental/litellm/uv.lock+++ b/examples/experimental/litellm/uv.lock@@ -69,25 +69,6 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" },]-[[package]]-name = "anthropic"-version = "0.120.2"-source = { registry = "https://pypi.org/simple" }-dependencies = [- { name = "anyio" },- { name = "distro" },- { name = "docstring-parser" },- { name = "httpx" },- { name = "jiter" },- { name = "pydantic" },- { name = "sniffio" },- { name = "typing-extensions" },-]-sdist = { url = "https://files.pythonhosted.org/packages/d7/10/4ca013cb166f226bd89e0aeb0fcaff94f45ddf716d4925ce89475d3c587b/anthropic-0.120.2.tar.gz", hash = "sha256:9722efc10c27a30a69f5338ddacdb35bc6a64297a4e4ba729bf83af873d5fb3a", size = 1008421, upload-time = "2026-07-28T17:38:26.986Z" }-wheels = [- { url = "https://files.pythonhosted.org/packages/63/af/0f5db57b9397a0f3b7fc204cbef143401a7cadaf982330f97f1ce3d39f34/anthropic-0.120.2-py3-none-any.whl", hash = "sha256:0f0bc2b381dc0eb41c8d886b815d79c2041cd2374f83aed36f574b6dc9c579c1", size = 1022851, upload-time = "2026-07-28T17:38:25.466Z" },-]-[[package]]name = "anyio"version = "4.14.2"@@ -171,15 +152,6 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" },]-[[package]]-name = "docstring-parser"-version = "0.18.0"-source = { registry = "https://pypi.org/simple" }-sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" }-wheels = [- { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" },-]-[[package]]name = "fastuuid"version = "0.14.0"@@ -484,46 +456,24 @@ wheels = [name = "nemo-switchyard"version = "0.2.0"source = { editable = "../../../" }-dependencies = [- { name = "anthropic" },- { name = "httpx" },- { name = "openai" },- { name = "pydantic" },-][package.metadata]-requires-dist = [- { name = "anthropic", specifier = ">=0.99.0,<1.0" },- { name = "ddtrace", marker = "extra == 'tracing'", specifier = ">=2.9,<4" },- { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.136.1,<1.0" },- { name = "httpx", specifier = ">=0.28.1,<1.0" },- { name = "nemo-switchyard", extras = ["server", "cli", "tracing", "affinity-redis"], marker = "extra == 'all'" },- { name = "openai", specifier = ">=2.7,<3.0" },- { name = "prompt-toolkit", marker = "extra == 'cli'", specifier = ">=3.0.52,<4.0" },- { name = "pydantic", specifier = ">=2.13.3,<3.0" },- { name = "redis", marker = "extra == 'affinity-redis'", specifier = ">=5.0.1,<6" },- { name = "sse-starlette", marker = "extra == 'server'", specifier = ">=3.4.1,<4.0" },- { name = "uvicorn", extras = ["standard"], marker = "extra == 'server'", specifier = ">=0.46.0,<1.0" },-]-provides-extras = ["server", "cli", "tracing", "affinity-redis", "all"]+requires-dist = [{ name = "prompt-toolkit", marker = "extra == 'cli'", specifier = ">=3.0.52,<4.0" }]+provides-extras = ["cli"][package.metadata.requires-dev]dev = [{ name = "harbor", marker = "python_full_version >= '3.12'", git = "https://github.com/harbor-framework/harbor.git?rev=v0.6.4" },- { name = "httpx", specifier = ">=0.28.1,<1.0" },{ name = "maturin", specifier = ">=1.9,<2.0" },{ name = "mypy", specifier = ">=1.20.2,<2.0" },- { name = "nemo-switchyard", extras = ["server"] },- { name = "prometheus-client", specifier = ">=0.21.0,<1.0" },+ { name = "nemo-switchyard", extras = ["cli"] },{ name = "pytest", specifier = ">=9.0.3,<10.0" },{ name = "pytest-asyncio", specifier = ">=1.3.0,<2.0" },- { name = "pytest-cov", specifier = ">=7.1.0,<8.0" },{ name = "pytest-markdown-docs", specifier = ">=0.9.2" },- { name = "pytest-mock", specifier = ">=3.15.1,<4.0" },{ name = "pytest-timeout", specifier = ">=2.4.0,<3.0" },- { name = "respx", specifier = ">=0.23.1,<1.0" },+ { name = "pyyaml", specifier = ">=6.0.3,<7.0" },{ name = "ruff", specifier = ">=0.15.12,<1.0" },- { name = "socksio", specifier = ">=1.0.0,<2.0" },+ { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0,<3.0" },]docs = [{ name = "mkdocs", specifier = ">=1.6.0,<2.0" },diff --git a/switchyard/libsy/__init__.py b/switchyard/libsy/__init__.pyindex 6d2b5ba74..78a535d01 100644--- a/switchyard/libsy/__init__.py+++ b/switchyard/libsy/__init__.py@@ -5,6 +5,7 @@from switchyard_rust.libsy import (Algorithm,+ ContextWindowExceededError,Decision,LibsyError,LlmFallback,@@ -17,6 +18,7 @@ from . import algorithms as algorithms__all__ = ["Algorithm",+ "ContextWindowExceededError","Decision","LibsyError","LlmFallback",diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.pyindex 244b1dd02..5dd607566 100644--- a/switchyard_rust/libsy.py+++ b/switchyard_rust/libsy.py@@ -13,6 +13,7 @@ from switchyard_rust._native import load_native_EXPORTS = frozenset({"Algorithm",+ "ContextWindowExceededError","Decision","LibsyError","LlmFallback",@@ -32,6 +33,8 @@ if TYPE_CHECKING:class LibsyError(RuntimeError): ...+ class ContextWindowExceededError(RuntimeError): ...+@finalclass Decision:"""A semantic routing choice produced by an algorithm."""diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.pyindex eb1d6c8ce..988454635 100644--- a/tests/test_libsy_minimal_bindings.py+++ b/tests/test_libsy_minimal_bindings.py@@ -9,6 +9,7 @@ import pytestfrom switchyard.libsy import (Algorithm,+ ContextWindowExceededError,Decision,LibsyError,Step,@@ -273,3 +274,23 @@ async def test_client_failure_becomes_libsy_error() -> None:with pytest.raises(LibsyError, match="client failed"):await run_algorithm(algorithm, {"broken": FailingClient()})+++async def test_context_window_failure_falls_back_to_the_next_model() -> None:+ class OverflowClient:+ async def call(self, request: dict[str, Any]) -> dict[str, Any]:+ raise ContextWindowExceededError("request exceeds context window")++ algorithm = algorithms.stage_router(+ "strong",+ "fast",+ picker="efficient_first",+ confidence_threshold=0.5,+ )+ decisions, response = await run_algorithm(+ algorithm,+ {"fast": OverflowClient(), "strong": EchoClient("strong")},+ )++ assert [decision.selected_model_id for decision in decisions] == ["fast", "strong"]+ assert response["model"] == "strong"