Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(python): align libsy streaming contracts (#479)
nachiketb-nvidiad70ba790d528
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rsindex 4546c549d..407a2080b 100644--- a/crates/switchyard-py/src/libsy_bindings.rs+++ b/crates/switchyard-py/src/libsy_bindings.rs@@ -19,7 +19,8 @@ use switchyard_libsy::{StepStream, TaskClassifierConfig,};use switchyard_protocol::{- AggLlmResponse, LlmClientError, LlmResponse, Metadata, ModelId, Request, Response,+ LlmClientError, LlmResponse, LlmResponseStream, LlmResponseStreamEvent, Metadata, ModelId,+ Request, Response,};use tokio::sync::Mutex;@@ -335,6 +336,84 @@ impl PyLlmFallback {}}+/// A normalized aggregate response or a live stream of normalized response events.+#[pyclass(name = "LlmResponse", module = "switchyard.libsy", frozen)]+enum PyLlmResponse {+ /// A fully buffered normalized response dictionary.+ #[pyo3(constructor = (response))]+ Agg { response: Py<PyAny> },+ /// An async iterator of normalized response-event dictionaries.+ #[pyo3(constructor = (stream))]+ Stream { stream: Py<PyAny> },+}++impl PyLlmResponse {+ fn to_core(&self, py: Python<'_>, model: ModelId) -> PyResult<LlmResponse> {+ match self {+ Self::Agg { response } => from_python(response.bind(py)).map(LlmResponse::Agg),+ Self::Stream { stream } => {+ python_response_stream(py, stream.clone_ref(py), model).map(LlmResponse::Stream)+ }+ }+ }+}++fn ffi_error(error: PyErr) -> LlmClientError {+ LlmClientError::Ffi {+ source: Box::new(error),+ }+}++fn python_client_error(py: Python<'_>, error: PyErr, model: &ModelId) -> LlmClientError {+ if error.is_instance_of::<ContextWindowExceededError>(py) {+ LlmClientError::ContextWindowExceeded {+ model: model.clone(),+ message: error.value(py).to_string(),+ }+ } else {+ ffi_error(error)+ }+}++fn python_response_stream(+ py: Python<'_>,+ stream: Py<PyAny>,+ model: ModelId,+) -> PyResult<LlmResponseStream> {+ let iterator = stream.bind(py).call_method0("__aiter__")?.unbind();+ // Rust polls on Tokio, so retain the Python task's event loop and context for every item.+ let locals = pyo3_async_runtimes::tokio::get_current_locals(py)?;+ let stream = futures::stream::unfold(Some((iterator, locals, model)), |state| async move {+ let (iterator, locals, model) = state?;+ let next = Python::attach(|py| {+ pyo3_async_runtimes::into_future_with_locals(+ &locals,+ iterator.bind(py).call_method0("__anext__")?,+ )+ });+ match next {+ Ok(next) => match next.await {+ Ok(item) => {+ let event =+ Python::attach(|py| from_python::<LlmResponseStreamEvent>(item.bind(py)))+ .map_err(ffi_error);+ Some((event, Some((iterator, locals, model))))+ }+ Err(error) => {+ if Python::attach(|py| error.is_instance_of::<PyStopAsyncIteration>(py)) {+ None+ } else {+ let error = Python::attach(|py| python_client_error(py, error, &model));+ Some((Err(error), None))+ }+ }+ },+ Err(error) => Some((Err(ffi_error(error)), None)),+ }+ });+ Ok(Box::pin(stream))+}+/// One model call yielded by [`PyAlgorithm::run_stream`].#[pyclass(name = "ModelCall", module = "switchyard.libsy")]struct PyModelCall {@@ -382,13 +461,23 @@ impl PyModelCall {self.models.clone()}- /// Fulfill this call with an aggregate normalized response dictionary.- fn respond(&mut self, response: &Bound<'_, PyAny>) -> PyResult<()> {- let aggregate = from_python::<AggLlmResponse>(response)?;+ /// Fulfill this call with a normalized aggregate or streamed response.+ fn respond(&mut self, py: Python<'_>, response: PyRef<'_, PyLlmResponse>) -> PyResult<()> {+ let model = self+ .inner+ .as_ref()+ .ok_or_else(|| py_libsy_error("model call has already been completed"))?+ .request+ .llm_request+ .model+ .as_ref()+ .map(ModelId::new)+ .ok_or_else(|| py_libsy_error("model call request is missing its selected model"))?;+ let llm_response = response.to_core(py, model)?;let call = self.take()?;let metadata = call.request.metadata.clone();call.respond(Ok(Response {- llm_response: LlmResponse::Agg(aggregate),+ llm_response,metadata,})).map_err(py_libsy_error)@@ -399,23 +488,18 @@ impl PyModelCall {if !error.is_instance_of::<PyBaseException>() {return Err(PyTypeError::new_err("error must derive from BaseException"));}+ let target = self+ .inner+ .as_ref()+ .ok_or_else(|| py_libsy_error("model call has already been completed"))?+ .request+ .llm_request+ .model+ .as_ref()+ .map(ModelId::new)+ .ok_or_else(|| py_libsy_error("model call request is missing its selected model"))?;let call = self.take()?;- let target = call- .models- .first()- .cloned()- .ok_or(RustLibsyError::NoTargets)- .map_err(py_libsy_error)?;- let source = if error.is_instance_of::<ContextWindowExceededError>() {- LlmClientError::ContextWindowExceeded {- model: target.clone(),- message: error.str()?.to_string_lossy().into_owned(),- }- } else {- LlmClientError::Ffi {- source: Box::new(PyErr::from_value(error.clone())),- }- };+ let source = python_client_error(error.py(), PyErr::from_value(error.clone()), &target);call.respond(Err(RustLibsyError::client_call(target, source))).map_err(py_libsy_error)}@@ -450,7 +534,7 @@ impl PyRoutingOutcome {self.request.clone_ref(py)}- /// An answer produced while routing, when one already exists.+ /// An aggregate or streamed answer produced while routing, when one already exists.#[getter]fn response(&self, py: Python<'_>) -> Option<Py<PyAny>> {self.response@@ -459,6 +543,50 @@ impl PyRoutingOutcome {}}+/// Async Python iterator over one normalized Rust response stream.+#[pyclass(name = "_LlmResponseStream", module = "switchyard.libsy", frozen)]+struct PyLlmResponseStream {+ inner: Arc<Mutex<LlmResponseStream>>,+}++#[pymethods]+impl PyLlmResponseStream {+ fn __aiter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> {+ slf+ }++ fn __anext__<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {+ let stream = Arc::clone(&self.inner);+ pyo3_async_runtimes::tokio::future_into_py(py, async move {+ match stream.lock().await.next().await {+ Some(Ok(event)) => Python::attach(|py| to_python(py, &event)),+ Some(Err(error)) => Err(py_libsy_error(error)),+ None => Err(PyStopAsyncIteration::new_err(())),+ }+ })+ }+}++fn response_to_python(py: Python<'_>, response: LlmResponse) -> PyResult<Py<PyAny>> {+ let response = match response {+ LlmResponse::Agg(response) => PyLlmResponse::Agg {+ response: to_python(py, &response)?,+ },+ LlmResponse::Stream(stream) => PyLlmResponse::Stream {+ stream: Py::new(+ py,+ PyLlmResponseStream {+ inner: Arc::new(Mutex::new(stream)),+ },+ )?+ .into_any(),+ },+ };+ response+ .into_pyobject(py)+ .map(|response| response.unbind().into_any())+}+/// One item yielded by a Python algorithm stream.#[pyclass(name = "Step", module = "switchyard.libsy", frozen)]enum PyStep {@@ -485,7 +613,7 @@ impl PyRunStream {pyo3_async_runtimes::tokio::future_into_py(py, async move {let step = stream.lock().await.next().await;match step {- Some(Ok(step)) => step_to_python(step).await,+ Some(Ok(step)) => step_to_python(step),Some(Err(error)) => Err(py_libsy_error(error)),None => Err(PyStopAsyncIteration::new_err(())),}@@ -533,7 +661,7 @@ impl PyAlgorithm {}}-async fn step_to_python(step: RustStep) -> PyResult<PyStep> {+fn step_to_python(step: RustStep) -> PyResult<PyStep> {match step {RustStep::CallModel(call) => Python::attach(|py| {Ok(PyStep::CallModel {@@ -547,16 +675,6 @@ async fn step_to_python(step: RustStep) -> PyResult<PyStep> {request,response,} = *outcome;- let response = match response {- Some(response) => Some(- response- .llm_response- .into_agg()- .await- .map_err(py_libsy_error)?,- ),- None => None,- };Python::attach(|py| {Ok(PyStep::Done {outcome: Py::new(@@ -569,8 +687,7 @@ async fn step_to_python(step: RustStep) -> PyResult<PyStep> {.collect(),request: to_python(py, &request.llm_request)?,response: response- .as_ref()- .map(|response| to_python(py, response))+ .map(|response| response_to_python(py, response.llm_response)).transpose()?,},)?,@@ -728,6 +845,8 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {libsy_module.add_class::<PyEscalationClassifierConfig>()?;libsy_module.add_class::<PyLlmClassifierConfig>()?;libsy_module.add_class::<PyLlmFallback>()?;+ libsy_module.add_class::<PyLlmResponse>()?;+ libsy_module.add_class::<PyLlmResponseStream>()?;libsy_module.add_class::<PyModelCall>()?;libsy_module.add_class::<PyRunStream>()?;libsy_module.add_class::<PyRoutingOutcome>()?;diff --git a/examples/experimental/litellm/README.md b/examples/experimental/litellm/README.mdindex 836cca915..084ce55ec 100644--- a/examples/experimental/litellm/README.md+++ b/examples/experimental/litellm/README.md@@ -107,7 +107,7 @@ normalized request shape and routing policy:```pythonimport asyncio-from switchyard.libsy import Step, algorithms+from switchyard.libsy import LlmResponse, Step, algorithmsfrom switchyard_litellm import LiteLLMSyClient@@ -161,17 +161,23 @@ async def main() -> None:try:async for step in router.run_stream(request):match step:- case Step.Decision(decision):- print("Decision:", decision.selected_model_id, decision.reasoning)case Step.CallModel(call):try:response = await client.call(call.request)except Exception as error:call.fail(error)else:- call.respond(response)- case Step.Done(response):- print("Response:", response)+ call.respond(LlmResponse.Agg(response))+ case Step.Done(outcome):+ print("Decision:", outcome.selected_model_id)+ match outcome.response:+ case LlmResponse.Agg(response):+ print("Response:", response)+ case LlmResponse.Stream(stream):+ async for event in stream:+ print("Response event:", event)+ case None:+ print("Response:", await client.call(outcome.request))finally:await client.aclose()diff --git a/examples/experimental/litellm/example.py b/examples/experimental/litellm/example.pyindex df2691865..96567d7e2 100644--- a/examples/experimental/litellm/example.py+++ b/examples/experimental/litellm/example.py@@ -8,7 +8,7 @@ import asynciofrom switchyard_litellm import LiteLLMSyClient-from switchyard.libsy import Step, algorithms+from switchyard.libsy import LlmResponse, Step, algorithmsdef sy_request() -> dict[str, object]:@@ -71,17 +71,23 @@ async def main() -> None:try:async for step in router.run_stream(sy_request()):match step:- case Step.Decision(decision):- print("Decision:", decision.selected_model_id, decision.reasoning)case Step.CallModel(call):try:response = await client.call(call.request)except Exception as error:call.fail(error)else:- call.respond(response)- case Step.Done(response):- print("Response:", response)+ call.respond(LlmResponse.Agg(response))+ case Step.Done(outcome):+ print("Decision:", outcome.selected_model_id)+ match outcome.response:+ case LlmResponse.Agg(response):+ print("Response:", response)+ case LlmResponse.Stream(stream):+ async for event in stream:+ print("Response event:", event)+ case None:+ print("Response:", await client.call(outcome.request))finally:await client.aclose()diff --git a/examples/experimental/litellm/tests/test_e2e.py b/examples/experimental/litellm/tests/test_e2e.pyindex ad5504681..9586eea9b 100644--- a/examples/experimental/litellm/tests/test_e2e.py+++ b/examples/experimental/litellm/tests/test_e2e.py@@ -11,7 +11,7 @@ from pathlib import Pathimport pytestfrom switchyard_litellm import LiteLLMSyClient-from switchyard.libsy import Algorithm, Decision, Step, algorithms+from switchyard.libsy import Algorithm, LlmResponse, Step, algorithmsPACKAGE_ROOT = Path(__file__).resolve().parents[1]@@ -147,17 +147,20 @@ async def _run_router(router: Algorithm,request: dict[str, object],client: LiteLLMSyClient,-) -> tuple[list[Decision], dict[str, object]]:- decisions: list[Decision] = []+) -> tuple[str, dict[str, object]]:async for step in router.run_stream(request):match step:- case Step.Decision(decision):- decisions.append(decision)case Step.CallModel(call):- call.respond(await client.call(call.request))- case Step.Done(response):- return decisions, response- raise AssertionError("algorithm stream ended without a response")+ call.respond(LlmResponse.Agg(await client.call(call.request)))+ case Step.Done(outcome):+ match outcome.response:+ case LlmResponse.Agg(response):+ return outcome.selected_model_id, response+ case LlmResponse.Stream(_):+ raise AssertionError("LiteLLM example expects buffered responses")+ case None:+ return outcome.selected_model_id, await client.call(outcome.request)+ raise AssertionError("algorithm stream ended without an outcome")@pytest.mark.e2e@@ -180,8 +183,8 @@ async def test_stage_router_calls_both_real_openrouter_models(finally:await client.aclose()- assert [item.selected_model_id for item in strong_trace] == ["strong"]- assert [item.selected_model_id for item in fast_trace] == ["fast"]+ assert strong_trace == "strong"+ assert fast_trace == "fast"for response in (strong_response, fast_response):text = response["outputs"][0]["content"][0]["text"]assert isinstance(text, str)diff --git a/examples/experimental/litellm/tests/test_stage_routing.py b/examples/experimental/litellm/tests/test_stage_routing.pyindex 54a1e051c..e6fb84877 100644--- a/examples/experimental/litellm/tests/test_stage_routing.py+++ b/examples/experimental/litellm/tests/test_stage_routing.py@@ -7,7 +7,7 @@ import httpximport respxfrom switchyard_litellm import LiteLLMSyClient-from switchyard.libsy import Algorithm, Decision, Step, algorithms+from switchyard.libsy import Algorithm, LlmResponse, Step, algorithmsBASE_URL = "http://gateway.test/v1"@@ -77,17 +77,20 @@ async def _run_router(router: Algorithm,request: dict[str, object],client: LiteLLMSyClient,-) -> tuple[list[Decision], dict[str, object]]:- decisions: list[Decision] = []+) -> tuple[str, dict[str, object]]:async for step in router.run_stream(request):match step:- case Step.Decision(decision):- decisions.append(decision)case Step.CallModel(call):- call.respond(await client.call(call.request))- case Step.Done(response):- return decisions, response- raise AssertionError("algorithm stream ended without a response")+ call.respond(LlmResponse.Agg(await client.call(call.request)))+ case Step.Done(outcome):+ match outcome.response:+ case LlmResponse.Agg(response):+ return outcome.selected_model_id, response+ case LlmResponse.Stream(_):+ raise AssertionError("LiteLLM example expects buffered responses")+ case None:+ return outcome.selected_model_id, await client.call(outcome.request)+ raise AssertionError("algorithm stream ended without an outcome")@respx.mock@@ -109,17 +112,17 @@ async def test_stage_router_drives_both_litellm_models() -> None:recent_window=3,)try:- fast_decisions, fast_response = await _run_router(+ fast_target, fast_response = await _run_router(router, request_body(), client)- strong_decisions, strong_response = await _run_router(+ strong_target, strong_response = await _run_router(router, request_body(critical_error=True), client)finally:await client.aclose()- assert [item.selected_model_id for item in fast_decisions] == ["fast"]- assert [item.selected_model_id for item in strong_decisions] == ["strong"]+ assert fast_target == "fast"+ assert strong_target == "strong"assert fast_response["outputs"][0]["content"][0]["text"] == "fast"assert strong_response["outputs"][0]["content"][0]["text"] == "strong"assert seen == ["fast", "strong"]diff --git a/examples/libsy.py b/examples/libsy.pyindex 3a1536331..471ab5b55 100644--- a/examples/libsy.py+++ b/examples/libsy.py@@ -5,9 +5,9 @@"""Drive a libsy algorithm stream from Python."""import asyncio-from collections.abc import Mapping+from collections.abc import AsyncIterator, Mapping-from switchyard.libsy import Step, algorithms+from switchyard.libsy import LlmResponse, Step, algorithmsclass EchoClient:@@ -17,19 +17,43 @@ class EchoClient:self,request: Mapping[str, object],model: str,- ) -> Mapping[str, object]:- return {- "model": model,- "outputs": [- {"role": "assistant", "content": [{"type": "text", "text": "Hello"}]}- ],- }+ ) -> LlmResponse.Agg | LlmResponse.Stream:+ if request.get("stream") is True:++ async def events() -> AsyncIterator[Mapping[str, object]]:+ yield {+ "preservation": None,+ "normalized": [{"MessageStart": {"id": "echo", "model": model}}],+ }+ yield {+ "preservation": None,+ "normalized": [{"TextDelta": {"index": 0, "text": "Hello"}}],+ }+ yield {+ "preservation": None,+ "normalized": [{"MessageStop": {"reason": "end_turn"}}],+ }++ return LlmResponse.Stream(events())++ return LlmResponse.Agg(+ {+ "model": model,+ "outputs": [+ {+ "role": "assistant",+ "content": [{"type": "text", "text": "Hello"}],+ }+ ],+ }+ )async def main() -> None:"""Run random routing and serve its selected target."""request = {"model": "auto",+ "stream": True,"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}],}client = EchoClient()@@ -41,13 +65,20 @@ async def main() -> None:async for step in algorithm.run_stream(request):match step:- case Step.Decision(decision):- print("Decision:", decision.selected_model_id, decision.reasoning)case Step.CallModel(call):- model = call.decision.selected_model_id- call.respond(await client.call(call.request, model))- case Step.Done(response):- print("Response:", response)+ call.respond(await client.call(call.request, call.models[0]))+ case Step.Done(outcome):+ print("Decision:", outcome.selected_model_id)+ response = outcome.response or await client.call(+ outcome.request,+ outcome.selected_model_id,+ )+ match response:+ case LlmResponse.Agg(aggregate_response):+ print("Response:", aggregate_response)+ case LlmResponse.Stream(response_stream):+ async for event in response_stream:+ print("Response event:", event)if __name__ == "__main__":diff --git a/switchyard/libsy/__init__.py b/switchyard/libsy/__init__.pyindex d6c2d243b..b4817dc87 100644--- a/switchyard/libsy/__init__.py+++ b/switchyard/libsy/__init__.py@@ -11,6 +11,7 @@ from switchyard_rust.libsy import (LibsyError,LlmClassifierConfig,LlmFallback,+ LlmResponse,ModelCall,RoutingOutcome,Step,@@ -27,6 +28,7 @@ __all__ = ["LibsyError","LlmClassifierConfig","LlmFallback",+ "LlmResponse","ModelCall","RoutingOutcome","Step",diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.pyindex c7037717b..cb15172cc 100644--- a/switchyard_rust/libsy.py+++ b/switchyard_rust/libsy.py@@ -19,6 +19,7 @@ _EXPORTS = frozenset("LibsyError","LlmClassifierConfig","LlmFallback",+ "LlmResponse","ModelCall","RoutingOutcome","Step",@@ -39,6 +40,23 @@ if TYPE_CHECKING:class ContextWindowExceededError(RuntimeError): ...+ class LlmResponse:+ """A normalized aggregate response or live normalized event stream."""++ @final+ class Agg:+ __match_args__: ClassVar[tuple[Literal["response"]]] = ("response",)+ response: dict[str, object]++ def __init__(self, response: Mapping[str, object]) -> None: ...++ @final+ class Stream:+ __match_args__: ClassVar[tuple[Literal["stream"]]] = ("stream",)+ stream: AsyncIterator[dict[str, object]]++ def __init__(self, stream: AsyncIterator[Mapping[str, object]]) -> None: ...+@finalclass CustomClassifierConfig:"""Configure schema-validated routing across named targets.@@ -89,7 +107,7 @@ if TYPE_CHECKING:@propertydef models(self) -> list[str]: ...- def respond(self, response: Mapping[str, object]) -> None: ...+ def respond(self, response: LlmResponse.Agg | LlmResponse.Stream) -> None: ...def fail(self, error: BaseException) -> None: ...@@ -105,7 +123,7 @@ if TYPE_CHECKING:def request(self) -> dict[str, object]: ...@property- def response(self) -> dict[str, object] | None: ...+ def response(self) -> LlmResponse.Agg | LlmResponse.Stream | None: ...class Step:@finaldiff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.pyindex ae2a9c7b4..07dd3cda7 100644--- a/tests/test_libsy_minimal_bindings.py+++ b/tests/test_libsy_minimal_bindings.py@@ -3,6 +3,7 @@"""Tests for the dictionary-based libsy Python API."""+from collections.abc import AsyncIteratorfrom typing import Anyimport pytest@@ -12,6 +13,7 @@ from switchyard.libsy import (ContextWindowExceededError,CustomClassifierConfig,LlmClassifierConfig,+ LlmResponse,RoutingOutcome,Step,TaskClassifierConfig,@@ -72,11 +74,15 @@ async def run_algorithm(call.fail(error)breakelse:- call.respond(response)+ call.respond(LlmResponse.Agg(response))breakcase Step.Done(outcome):if outcome.response is not None:- return outcome.selected_model_id, outcome.response+ match outcome.response:+ case LlmResponse.Agg(response):+ return outcome.selected_model_id, response+ case LlmResponse.Stream(_):+ raise AssertionError("test helper expected an aggregate response")candidates = [outcome.selected_model_id, *outcome.fallback_models]for index, target in enumerate(candidates):candidate_request = {**outcome.request, "model": target}@@ -117,6 +123,42 @@ async def test_random_streams_complex_steps_and_accepts_a_dictionary_response()assert response["outputs"][0]["content"] == [{"type": "text", "text": "fast"}]+async def test_routing_call_accepts_a_streamed_response() -> None:+ async def events() -> AsyncIterator[dict[str, object]]:+ for chunk in [+ {"MessageStart": {"id": "response-1", "model": "judge"}},+ {"TextDelta": {"index": 0, "text": '{"target":"balanced"}'}},+ {"MessageStop": {"reason": "end_turn"}},+ ]:+ yield {"preservation": None, "normalized": [chunk]}++ schema = {+ "type": "object",+ "additionalProperties": False,+ "required": ["target"],+ "properties": {"target": {"type": "string", "enum": ["fast", "balanced"]}},+ }+ algorithm = algorithms.llm_classifier(+ LlmClassifierConfig.custom(+ "judge",+ [("fast", "model-a"), ("balanced", "model-b")],+ default_target="fast",+ config=CustomClassifierConfig("Choose a target.", schema, "/target"),+ )+ )+ outcome: RoutingOutcome | None = None++ async for step in algorithm.run_stream(request_body()):+ match step:+ case Step.CallModel(call):+ call.respond(LlmResponse.Stream(events()))+ case Step.Done(done):+ outcome = done++ assert outcome is not None+ assert outcome.selected_model_id == "model-b"++async def test_classifier_config_accepts_a_prompt_override() -> None:"""Verify that a configured classifier prompt is rendered for the judge."""