Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(python): align libsy streaming contracts (#479)

nachiketb-nvidiad70ba790d528
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index 4546c549d..407a2080b 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -19,7 +19,8 @@ use switchyard_libsy::{
StepStream, TaskClassifierConfig,
};
use switchyard_protocol::{
- AggLlmResponse, LlmClientError, LlmResponse, Metadata, ModelId, Request, Response,
+ LlmClientError, LlmResponse, LlmResponseStream, LlmResponseStreamEvent, Metadata, ModelId,
+ Request, Response,
};
use tokio::sync::Mutex;
@@ -335,6 +336,84 @@ impl PyLlmFallback {
}
}
+/// A normalized aggregate response or a live stream of normalized response events.
+#[pyclass(name = "LlmResponse", module = "switchyard.libsy", frozen)]
+enum PyLlmResponse {
+ /// A fully buffered normalized response dictionary.
+ #[pyo3(constructor = (response))]
+ Agg { response: Py<PyAny> },
+ /// An async iterator of normalized response-event dictionaries.
+ #[pyo3(constructor = (stream))]
+ Stream { stream: Py<PyAny> },
+}
+
+impl PyLlmResponse {
+ fn to_core(&self, py: Python<'_>, model: ModelId) -> PyResult<LlmResponse> {
+ match self {
+ Self::Agg { response } => from_python(response.bind(py)).map(LlmResponse::Agg),
+ Self::Stream { stream } => {
+ python_response_stream(py, stream.clone_ref(py), model).map(LlmResponse::Stream)
+ }
+ }
+ }
+}
+
+fn ffi_error(error: PyErr) -> LlmClientError {
+ LlmClientError::Ffi {
+ source: Box::new(error),
+ }
+}
+
+fn python_client_error(py: Python<'_>, error: PyErr, model: &ModelId) -> LlmClientError {
+ if error.is_instance_of::<ContextWindowExceededError>(py) {
+ LlmClientError::ContextWindowExceeded {
+ model: model.clone(),
+ message: error.value(py).to_string(),
+ }
+ } else {
+ ffi_error(error)
+ }
+}
+
+fn python_response_stream(
+ py: Python<'_>,
+ stream: Py<PyAny>,
+ model: ModelId,
+) -> PyResult<LlmResponseStream> {
+ let iterator = stream.bind(py).call_method0("__aiter__")?.unbind();
+ // Rust polls on Tokio, so retain the Python task's event loop and context for every item.
+ let locals = pyo3_async_runtimes::tokio::get_current_locals(py)?;
+ let stream = futures::stream::unfold(Some((iterator, locals, model)), |state| async move {
+ let (iterator, locals, model) = state?;
+ let next = Python::attach(|py| {
+ pyo3_async_runtimes::into_future_with_locals(
+ &locals,
+ iterator.bind(py).call_method0("__anext__")?,
+ )
+ });
+ match next {
+ Ok(next) => match next.await {
+ Ok(item) => {
+ let event =
+ Python::attach(|py| from_python::<LlmResponseStreamEvent>(item.bind(py)))
+ .map_err(ffi_error);
+ Some((event, Some((iterator, locals, model))))
+ }
+ Err(error) => {
+ if Python::attach(|py| error.is_instance_of::<PyStopAsyncIteration>(py)) {
+ None
+ } else {
+ let error = Python::attach(|py| python_client_error(py, error, &model));
+ Some((Err(error), None))
+ }
+ }
+ },
+ Err(error) => Some((Err(ffi_error(error)), None)),
+ }
+ });
+ Ok(Box::pin(stream))
+}
+
/// One model call yielded by [`PyAlgorithm::run_stream`].
#[pyclass(name = "ModelCall", module = "switchyard.libsy")]
struct PyModelCall {
@@ -382,13 +461,23 @@ impl PyModelCall {
self.models.clone()
}
- /// Fulfill this call with an aggregate normalized response dictionary.
- fn respond(&mut self, response: &Bound<'_, PyAny>) -> PyResult<()> {
- let aggregate = from_python::<AggLlmResponse>(response)?;
+ /// Fulfill this call with a normalized aggregate or streamed response.
+ fn respond(&mut self, py: Python<'_>, response: PyRef<'_, PyLlmResponse>) -> PyResult<()> {
+ let model = self
+ .inner
+ .as_ref()
+ .ok_or_else(|| py_libsy_error("model call has already been completed"))?
+ .request
+ .llm_request
+ .model
+ .as_ref()
+ .map(ModelId::new)
+ .ok_or_else(|| py_libsy_error("model call request is missing its selected model"))?;
+ let llm_response = response.to_core(py, model)?;
let call = self.take()?;
let metadata = call.request.metadata.clone();
call.respond(Ok(Response {
- llm_response: LlmResponse::Agg(aggregate),
+ llm_response,
metadata,
}))
.map_err(py_libsy_error)
@@ -399,23 +488,18 @@ impl PyModelCall {
if !error.is_instance_of::<PyBaseException>() {
return Err(PyTypeError::new_err("error must derive from BaseException"));
}
+ let target = self
+ .inner
+ .as_ref()
+ .ok_or_else(|| py_libsy_error("model call has already been completed"))?
+ .request
+ .llm_request
+ .model
+ .as_ref()
+ .map(ModelId::new)
+ .ok_or_else(|| py_libsy_error("model call request is missing its selected model"))?;
let call = self.take()?;
- let target = call
- .models
- .first()
- .cloned()
- .ok_or(RustLibsyError::NoTargets)
- .map_err(py_libsy_error)?;
- let source = if error.is_instance_of::<ContextWindowExceededError>() {
- LlmClientError::ContextWindowExceeded {
- model: target.clone(),
- message: error.str()?.to_string_lossy().into_owned(),
- }
- } else {
- LlmClientError::Ffi {
- source: Box::new(PyErr::from_value(error.clone())),
- }
- };
+ let source = python_client_error(error.py(), PyErr::from_value(error.clone()), &target);
call.respond(Err(RustLibsyError::client_call(target, source)))
.map_err(py_libsy_error)
}
@@ -450,7 +534,7 @@ impl PyRoutingOutcome {
self.request.clone_ref(py)
}
- /// An answer produced while routing, when one already exists.
+ /// An aggregate or streamed answer produced while routing, when one already exists.
#[getter]
fn response(&self, py: Python<'_>) -> Option<Py<PyAny>> {
self.response
@@ -459,6 +543,50 @@ impl PyRoutingOutcome {
}
}
+/// Async Python iterator over one normalized Rust response stream.
+#[pyclass(name = "_LlmResponseStream", module = "switchyard.libsy", frozen)]
+struct PyLlmResponseStream {
+ inner: Arc<Mutex<LlmResponseStream>>,
+}
+
+#[pymethods]
+impl PyLlmResponseStream {
+ fn __aiter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> {
+ slf
+ }
+
+ fn __anext__<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
+ let stream = Arc::clone(&self.inner);
+ pyo3_async_runtimes::tokio::future_into_py(py, async move {
+ match stream.lock().await.next().await {
+ Some(Ok(event)) => Python::attach(|py| to_python(py, &event)),
+ Some(Err(error)) => Err(py_libsy_error(error)),
+ None => Err(PyStopAsyncIteration::new_err(())),
+ }
+ })
+ }
+}
+
+fn response_to_python(py: Python<'_>, response: LlmResponse) -> PyResult<Py<PyAny>> {
+ let response = match response {
+ LlmResponse::Agg(response) => PyLlmResponse::Agg {
+ response: to_python(py, &response)?,
+ },
+ LlmResponse::Stream(stream) => PyLlmResponse::Stream {
+ stream: Py::new(
+ py,
+ PyLlmResponseStream {
+ inner: Arc::new(Mutex::new(stream)),
+ },
+ )?
+ .into_any(),
+ },
+ };
+ response
+ .into_pyobject(py)
+ .map(|response| response.unbind().into_any())
+}
+
/// One item yielded by a Python algorithm stream.
#[pyclass(name = "Step", module = "switchyard.libsy", frozen)]
enum PyStep {
@@ -485,7 +613,7 @@ impl PyRunStream {
pyo3_async_runtimes::tokio::future_into_py(py, async move {
let step = stream.lock().await.next().await;
match step {
- Some(Ok(step)) => step_to_python(step).await,
+ Some(Ok(step)) => step_to_python(step),
Some(Err(error)) => Err(py_libsy_error(error)),
None => Err(PyStopAsyncIteration::new_err(())),
}
@@ -533,7 +661,7 @@ impl PyAlgorithm {
}
}
-async fn step_to_python(step: RustStep) -> PyResult<PyStep> {
+fn step_to_python(step: RustStep) -> PyResult<PyStep> {
match step {
RustStep::CallModel(call) => Python::attach(|py| {
Ok(PyStep::CallModel {
@@ -547,16 +675,6 @@ async fn step_to_python(step: RustStep) -> PyResult<PyStep> {
request,
response,
} = *outcome;
- let response = match response {
- Some(response) => Some(
- response
- .llm_response
- .into_agg()
- .await
- .map_err(py_libsy_error)?,
- ),
- None => None,
- };
Python::attach(|py| {
Ok(PyStep::Done {
outcome: Py::new(
@@ -569,8 +687,7 @@ async fn step_to_python(step: RustStep) -> PyResult<PyStep> {
.collect(),
request: to_python(py, &request.llm_request)?,
response: response
- .as_ref()
- .map(|response| to_python(py, response))
+ .map(|response| response_to_python(py, response.llm_response))
.transpose()?,
},
)?,
@@ -728,6 +845,8 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
libsy_module.add_class::<PyEscalationClassifierConfig>()?;
libsy_module.add_class::<PyLlmClassifierConfig>()?;
libsy_module.add_class::<PyLlmFallback>()?;
+ libsy_module.add_class::<PyLlmResponse>()?;
+ libsy_module.add_class::<PyLlmResponseStream>()?;
libsy_module.add_class::<PyModelCall>()?;
libsy_module.add_class::<PyRunStream>()?;
libsy_module.add_class::<PyRoutingOutcome>()?;
diff --git a/examples/experimental/litellm/README.md b/examples/experimental/litellm/README.md
index 836cca915..084ce55ec 100644
--- a/examples/experimental/litellm/README.md
+++ b/examples/experimental/litellm/README.md
@@ -107,7 +107,7 @@ normalized request shape and routing policy:
```python
import asyncio
-from switchyard.libsy import Step, algorithms
+from switchyard.libsy import LlmResponse, Step, algorithms
from switchyard_litellm import LiteLLMSyClient
@@ -161,17 +161,23 @@ async def main() -> None:
try:
async for step in router.run_stream(request):
match step:
- case Step.Decision(decision):
- print("Decision:", decision.selected_model_id, decision.reasoning)
case Step.CallModel(call):
try:
response = await client.call(call.request)
except Exception as error:
call.fail(error)
else:
- call.respond(response)
- case Step.Done(response):
- print("Response:", response)
+ call.respond(LlmResponse.Agg(response))
+ case Step.Done(outcome):
+ print("Decision:", outcome.selected_model_id)
+ match outcome.response:
+ case LlmResponse.Agg(response):
+ print("Response:", response)
+ case LlmResponse.Stream(stream):
+ async for event in stream:
+ print("Response event:", event)
+ case None:
+ print("Response:", await client.call(outcome.request))
finally:
await client.aclose()
diff --git a/examples/experimental/litellm/example.py b/examples/experimental/litellm/example.py
index df2691865..96567d7e2 100644
--- a/examples/experimental/litellm/example.py
+++ b/examples/experimental/litellm/example.py
@@ -8,7 +8,7 @@ import asyncio
from switchyard_litellm import LiteLLMSyClient
-from switchyard.libsy import Step, algorithms
+from switchyard.libsy import LlmResponse, Step, algorithms
def sy_request() -> dict[str, object]:
@@ -71,17 +71,23 @@ async def main() -> None:
try:
async for step in router.run_stream(sy_request()):
match step:
- case Step.Decision(decision):
- print("Decision:", decision.selected_model_id, decision.reasoning)
case Step.CallModel(call):
try:
response = await client.call(call.request)
except Exception as error:
call.fail(error)
else:
- call.respond(response)
- case Step.Done(response):
- print("Response:", response)
+ call.respond(LlmResponse.Agg(response))
+ case Step.Done(outcome):
+ print("Decision:", outcome.selected_model_id)
+ match outcome.response:
+ case LlmResponse.Agg(response):
+ print("Response:", response)
+ case LlmResponse.Stream(stream):
+ async for event in stream:
+ print("Response event:", event)
+ case None:
+ print("Response:", await client.call(outcome.request))
finally:
await client.aclose()
diff --git a/examples/experimental/litellm/tests/test_e2e.py b/examples/experimental/litellm/tests/test_e2e.py
index ad5504681..9586eea9b 100644
--- a/examples/experimental/litellm/tests/test_e2e.py
+++ b/examples/experimental/litellm/tests/test_e2e.py
@@ -11,7 +11,7 @@ from pathlib import Path
import pytest
from switchyard_litellm import LiteLLMSyClient
-from switchyard.libsy import Algorithm, Decision, Step, algorithms
+from switchyard.libsy import Algorithm, LlmResponse, Step, algorithms
PACKAGE_ROOT = Path(__file__).resolve().parents[1]
@@ -147,17 +147,20 @@ async def _run_router(
router: Algorithm,
request: dict[str, object],
client: LiteLLMSyClient,
-) -> tuple[list[Decision], dict[str, object]]:
- decisions: list[Decision] = []
+) -> tuple[str, dict[str, object]]:
async for step in router.run_stream(request):
match step:
- case Step.Decision(decision):
- decisions.append(decision)
case Step.CallModel(call):
- call.respond(await client.call(call.request))
- case Step.Done(response):
- return decisions, response
- raise AssertionError("algorithm stream ended without a response")
+ call.respond(LlmResponse.Agg(await client.call(call.request)))
+ case Step.Done(outcome):
+ match outcome.response:
+ case LlmResponse.Agg(response):
+ return outcome.selected_model_id, response
+ case LlmResponse.Stream(_):
+ raise AssertionError("LiteLLM example expects buffered responses")
+ case None:
+ return outcome.selected_model_id, await client.call(outcome.request)
+ raise AssertionError("algorithm stream ended without an outcome")
@pytest.mark.e2e
@@ -180,8 +183,8 @@ async def test_stage_router_calls_both_real_openrouter_models(
finally:
await client.aclose()
- assert [item.selected_model_id for item in strong_trace] == ["strong"]
- assert [item.selected_model_id for item in fast_trace] == ["fast"]
+ assert strong_trace == "strong"
+ assert fast_trace == "fast"
for response in (strong_response, fast_response):
text = response["outputs"][0]["content"][0]["text"]
assert isinstance(text, str)
diff --git a/examples/experimental/litellm/tests/test_stage_routing.py b/examples/experimental/litellm/tests/test_stage_routing.py
index 54a1e051c..e6fb84877 100644
--- a/examples/experimental/litellm/tests/test_stage_routing.py
+++ b/examples/experimental/litellm/tests/test_stage_routing.py
@@ -7,7 +7,7 @@ import httpx
import respx
from switchyard_litellm import LiteLLMSyClient
-from switchyard.libsy import Algorithm, Decision, Step, algorithms
+from switchyard.libsy import Algorithm, LlmResponse, Step, algorithms
BASE_URL = "http://gateway.test/v1"
@@ -77,17 +77,20 @@ async def _run_router(
router: Algorithm,
request: dict[str, object],
client: LiteLLMSyClient,
-) -> tuple[list[Decision], dict[str, object]]:
- decisions: list[Decision] = []
+) -> tuple[str, dict[str, object]]:
async for step in router.run_stream(request):
match step:
- case Step.Decision(decision):
- decisions.append(decision)
case Step.CallModel(call):
- call.respond(await client.call(call.request))
- case Step.Done(response):
- return decisions, response
- raise AssertionError("algorithm stream ended without a response")
+ call.respond(LlmResponse.Agg(await client.call(call.request)))
+ case Step.Done(outcome):
+ match outcome.response:
+ case LlmResponse.Agg(response):
+ return outcome.selected_model_id, response
+ case LlmResponse.Stream(_):
+ raise AssertionError("LiteLLM example expects buffered responses")
+ case None:
+ return outcome.selected_model_id, await client.call(outcome.request)
+ raise AssertionError("algorithm stream ended without an outcome")
@respx.mock
@@ -109,17 +112,17 @@ async def test_stage_router_drives_both_litellm_models() -> None:
recent_window=3,
)
try:
- fast_decisions, fast_response = await _run_router(
+ fast_target, fast_response = await _run_router(
router, request_body(), client
)
- strong_decisions, strong_response = await _run_router(
+ strong_target, strong_response = await _run_router(
router, request_body(critical_error=True), client
)
finally:
await client.aclose()
- assert [item.selected_model_id for item in fast_decisions] == ["fast"]
- assert [item.selected_model_id for item in strong_decisions] == ["strong"]
+ assert fast_target == "fast"
+ assert strong_target == "strong"
assert fast_response["outputs"][0]["content"][0]["text"] == "fast"
assert strong_response["outputs"][0]["content"][0]["text"] == "strong"
assert seen == ["fast", "strong"]
diff --git a/examples/libsy.py b/examples/libsy.py
index 3a1536331..471ab5b55 100644
--- a/examples/libsy.py
+++ b/examples/libsy.py
@@ -5,9 +5,9 @@
"""Drive a libsy algorithm stream from Python."""
import asyncio
-from collections.abc import Mapping
+from collections.abc import AsyncIterator, Mapping
-from switchyard.libsy import Step, algorithms
+from switchyard.libsy import LlmResponse, Step, algorithms
class EchoClient:
@@ -17,19 +17,43 @@ class EchoClient:
self,
request: Mapping[str, object],
model: str,
- ) -> Mapping[str, object]:
- return {
- "model": model,
- "outputs": [
- {"role": "assistant", "content": [{"type": "text", "text": "Hello"}]}
- ],
- }
+ ) -> LlmResponse.Agg | LlmResponse.Stream:
+ if request.get("stream") is True:
+
+ async def events() -> AsyncIterator[Mapping[str, object]]:
+ yield {
+ "preservation": None,
+ "normalized": [{"MessageStart": {"id": "echo", "model": model}}],
+ }
+ yield {
+ "preservation": None,
+ "normalized": [{"TextDelta": {"index": 0, "text": "Hello"}}],
+ }
+ yield {
+ "preservation": None,
+ "normalized": [{"MessageStop": {"reason": "end_turn"}}],
+ }
+
+ return LlmResponse.Stream(events())
+
+ return LlmResponse.Agg(
+ {
+ "model": model,
+ "outputs": [
+ {
+ "role": "assistant",
+ "content": [{"type": "text", "text": "Hello"}],
+ }
+ ],
+ }
+ )
async def main() -> None:
"""Run random routing and serve its selected target."""
request = {
"model": "auto",
+ "stream": True,
"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}],
}
client = EchoClient()
@@ -41,13 +65,20 @@ async def main() -> None:
async for step in algorithm.run_stream(request):
match step:
- case Step.Decision(decision):
- print("Decision:", decision.selected_model_id, decision.reasoning)
case Step.CallModel(call):
- model = call.decision.selected_model_id
- call.respond(await client.call(call.request, model))
- case Step.Done(response):
- print("Response:", response)
+ call.respond(await client.call(call.request, call.models[0]))
+ case Step.Done(outcome):
+ print("Decision:", outcome.selected_model_id)
+ response = outcome.response or await client.call(
+ outcome.request,
+ outcome.selected_model_id,
+ )
+ match response:
+ case LlmResponse.Agg(aggregate_response):
+ print("Response:", aggregate_response)
+ case LlmResponse.Stream(response_stream):
+ async for event in response_stream:
+ print("Response event:", event)
if __name__ == "__main__":
diff --git a/switchyard/libsy/__init__.py b/switchyard/libsy/__init__.py
index d6c2d243b..b4817dc87 100644
--- a/switchyard/libsy/__init__.py
+++ b/switchyard/libsy/__init__.py
@@ -11,6 +11,7 @@ from switchyard_rust.libsy import (
LibsyError,
LlmClassifierConfig,
LlmFallback,
+ LlmResponse,
ModelCall,
RoutingOutcome,
Step,
@@ -27,6 +28,7 @@ __all__ = [
"LibsyError",
"LlmClassifierConfig",
"LlmFallback",
+ "LlmResponse",
"ModelCall",
"RoutingOutcome",
"Step",
diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py
index c7037717b..cb15172cc 100644
--- a/switchyard_rust/libsy.py
+++ b/switchyard_rust/libsy.py
@@ -19,6 +19,7 @@ _EXPORTS = frozenset(
"LibsyError",
"LlmClassifierConfig",
"LlmFallback",
+ "LlmResponse",
"ModelCall",
"RoutingOutcome",
"Step",
@@ -39,6 +40,23 @@ if TYPE_CHECKING:
class ContextWindowExceededError(RuntimeError): ...
+ class LlmResponse:
+ """A normalized aggregate response or live normalized event stream."""
+
+ @final
+ class Agg:
+ __match_args__: ClassVar[tuple[Literal["response"]]] = ("response",)
+ response: dict[str, object]
+
+ def __init__(self, response: Mapping[str, object]) -> None: ...
+
+ @final
+ class Stream:
+ __match_args__: ClassVar[tuple[Literal["stream"]]] = ("stream",)
+ stream: AsyncIterator[dict[str, object]]
+
+ def __init__(self, stream: AsyncIterator[Mapping[str, object]]) -> None: ...
+
@final
class CustomClassifierConfig:
"""Configure schema-validated routing across named targets.
@@ -89,7 +107,7 @@ if TYPE_CHECKING:
@property
def models(self) -> list[str]: ...
- def respond(self, response: Mapping[str, object]) -> None: ...
+ def respond(self, response: LlmResponse.Agg | LlmResponse.Stream) -> None: ...
def fail(self, error: BaseException) -> None: ...
@@ -105,7 +123,7 @@ if TYPE_CHECKING:
def request(self) -> dict[str, object]: ...
@property
- def response(self) -> dict[str, object] | None: ...
+ def response(self) -> LlmResponse.Agg | LlmResponse.Stream | None: ...
class Step:
@final
diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py
index ae2a9c7b4..07dd3cda7 100644
--- a/tests/test_libsy_minimal_bindings.py
+++ b/tests/test_libsy_minimal_bindings.py
@@ -3,6 +3,7 @@
"""Tests for the dictionary-based libsy Python API."""
+from collections.abc import AsyncIterator
from typing import Any
import pytest
@@ -12,6 +13,7 @@ from switchyard.libsy import (
ContextWindowExceededError,
CustomClassifierConfig,
LlmClassifierConfig,
+ LlmResponse,
RoutingOutcome,
Step,
TaskClassifierConfig,
@@ -72,11 +74,15 @@ async def run_algorithm(
call.fail(error)
break
else:
- call.respond(response)
+ call.respond(LlmResponse.Agg(response))
break
case Step.Done(outcome):
if outcome.response is not None:
- return outcome.selected_model_id, outcome.response
+ match outcome.response:
+ case LlmResponse.Agg(response):
+ return outcome.selected_model_id, response
+ case LlmResponse.Stream(_):
+ raise AssertionError("test helper expected an aggregate response")
candidates = [outcome.selected_model_id, *outcome.fallback_models]
for index, target in enumerate(candidates):
candidate_request = {**outcome.request, "model": target}
@@ -117,6 +123,42 @@ async def test_random_streams_complex_steps_and_accepts_a_dictionary_response()
assert response["outputs"][0]["content"] == [{"type": "text", "text": "fast"}]
+async def test_routing_call_accepts_a_streamed_response() -> None:
+ async def events() -> AsyncIterator[dict[str, object]]:
+ for chunk in [
+ {"MessageStart": {"id": "response-1", "model": "judge"}},
+ {"TextDelta": {"index": 0, "text": '{"target":"balanced"}'}},
+ {"MessageStop": {"reason": "end_turn"}},
+ ]:
+ yield {"preservation": None, "normalized": [chunk]}
+
+ schema = {
+ "type": "object",
+ "additionalProperties": False,
+ "required": ["target"],
+ "properties": {"target": {"type": "string", "enum": ["fast", "balanced"]}},
+ }
+ algorithm = algorithms.llm_classifier(
+ LlmClassifierConfig.custom(
+ "judge",
+ [("fast", "model-a"), ("balanced", "model-b")],
+ default_target="fast",
+ config=CustomClassifierConfig("Choose a target.", schema, "/target"),
+ )
+ )
+ outcome: RoutingOutcome | None = None
+
+ async for step in algorithm.run_stream(request_body()):
+ match step:
+ case Step.CallModel(call):
+ call.respond(LlmResponse.Stream(events()))
+ case Step.Done(done):
+ outcome = done
+
+ assert outcome is not None
+ assert outcome.selected_model_id == "model-b"
+
+
async def test_classifier_config_accepts_a_prompt_override() -> None:
"""Verify that a configured classifier prompt is rendered for the judge."""