Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(python): add streaming translation bindings (#1232)

David Gardnerf6b3a640ee22
diff --git a/crates/switchyard-py/src/translation.rs b/crates/switchyard-py/src/translation.rs
index e2e78fb11..2740344b1 100644
--- a/crates/switchyard-py/src/translation.rs
+++ b/crates/switchyard-py/src/translation.rs
@@ -1,16 +1,23 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
+use std::sync::Arc;
+
+use futures::{StreamExt, stream::BoxStream};
use pyo3::create_exception;
-use pyo3::exceptions::PyException;
+use pyo3::exceptions::{PyException, PyStopAsyncIteration};
use pyo3::prelude::*;
-use pyo3::types::PyDict;
+use pyo3::types::{PyBytes, PyDict};
+use serde::Serialize;
use serde_json::{Value, json};
+use switchyard_protocol::{LlmClientError, LlmResponseStreamEvent, LlmStreamError};
use switchyard_translation::{
AggLlmResponse, LlmRequest, ProviderExtensions, TranslationEngine,
TranslationError as CoreTranslationError, TranslationOutput, TranslationPolicy,
};
+use tokio::sync::Mutex;
+
use crate::py_serde::{from_python, to_python};
create_exception!(
@@ -20,7 +27,7 @@ create_exception!(
"Translation failed. Attributes: kind (stable variant name), details (variant fields)."
);
-/// Translate buffered JSON requests and responses with the default policy.
+/// Translate provider requests, responses, and streams.
#[pyclass(name = "TranslationEngine", module = "switchyard.translation", frozen)]
struct PyTranslationEngine {
inner: TranslationEngine,
@@ -146,6 +153,246 @@ impl PyTranslationEngine {
});
translation_output(py, output)
}
+
+ /// Decode SSE byte chunks into an async iterator of IR event dictionaries.
+ fn decode_stream(
+ &self,
+ py: Python<'_>,
+ source: &str,
+ byte_stream: &Bound<'_, PyAny>,
+ ) -> PyResult<Py<PyAny>> {
+ let module = py.import("switchyard_rust._translation_stream")?;
+ let input = create_stream_input(&module, byte_stream)?;
+ let bytes = input_stream(&input, |item| {
+ Ok(item.cast::<PyBytes>()?.as_bytes().to_vec())
+ });
+ let stream = self
+ .inner
+ .decode_stream(source, bytes)
+ .map_err(|error| client_error(py, error))?;
+ output_stream(
+ &module,
+ input,
+ stream
+ .map(|item| item.map_err(LlmStreamError::Client))
+ .boxed(),
+ )
+ }
+
+ /// Encode IR events into an async iterator of provider event dictionaries.
+ #[pyo3(signature = (target, chunks, *, served_model=None, request_extensions=None))]
+ fn encode_stream(
+ &self,
+ py: Python<'_>,
+ target: &str,
+ chunks: &Bound<'_, PyAny>,
+ served_model: Option<String>,
+ request_extensions: Option<&Bound<'_, PyDict>>,
+ ) -> PyResult<Py<PyAny>> {
+ let extensions = request_extensions
+ .map(|extensions| from_python(extensions.as_any()))
+ .transpose()?
+ .unwrap_or_default();
+ let module = py.import("switchyard_rust._translation_stream")?;
+ let input = create_stream_input(&module, chunks)?;
+ let chunks = input_stream(&input, |item| {
+ from_python::<LlmResponseStreamEvent>(item.cast::<PyDict>()?.as_any())
+ });
+ let stream = self
+ .inner
+ .encode_stream_with_extensions(target, chunks, served_model, &extensions)
+ .map_err(|error| client_error(py, error))?;
+ output_stream(&module, input, stream)
+ }
+
+ /// Translate SSE byte chunks into target-format event dictionaries.
+ #[pyo3(signature = (source, target, byte_stream, *, served_model=None, request_extensions=None))]
+ fn translate_stream(
+ &self,
+ py: Python<'_>,
+ source: &str,
+ target: &str,
+ byte_stream: &Bound<'_, PyAny>,
+ served_model: Option<String>,
+ request_extensions: Option<&Bound<'_, PyDict>>,
+ ) -> PyResult<Py<PyAny>> {
+ let module = py.import("switchyard_rust._translation_stream")?;
+ let input = create_stream_input(&module, byte_stream)?;
+ let bytes = input_stream(&input, |item| {
+ Ok(item.cast::<PyBytes>()?.as_bytes().to_vec())
+ });
+ let decoded = self
+ .inner
+ .decode_stream(source, bytes)
+ .map_err(|error| client_error(py, error))?;
+ let extensions = request_extensions
+ .map(|extensions| from_python(extensions.as_any()))
+ .transpose()?
+ .unwrap_or_default();
+ let encoded = self
+ .inner
+ .encode_stream_with_extensions(target, decoded, served_model, &extensions)
+ .map_err(|error| client_error(py, error))?;
+ output_stream(&module, input, encoded)
+ }
+}
+
+// Each Python awaitable owns a handle while it polls; _TranslationStream permits one poll at a time.
+#[pyclass(frozen)]
+struct NativeStream {
+ inner: Arc<Mutex<BoxStream<'static, PyResult<Py<PyAny>>>>>,
+}
+
+#[pymethods]
+impl NativeStream {
+ fn __anext__<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
+ let inner = Arc::clone(&self.inner);
+ pyo3_async_runtimes::tokio::future_into_py(py, async move {
+ inner
+ .lock()
+ .await
+ .next()
+ .await
+ .unwrap_or_else(|| Err(PyStopAsyncIteration::new_err(())))
+ })
+ }
+}
+
+fn create_stream_input<'py>(
+ module: &Bound<'py, PyModule>,
+ source: &Bound<'py, PyAny>,
+) -> PyResult<Bound<'py, PyAny>> {
+ module.getattr("_StreamInput")?.call1((source,))
+}
+
+fn input_stream<T: Send + 'static>(
+ input: &Bound<'_, PyAny>,
+ convert: fn(&Bound<'_, PyAny>) -> PyResult<T>,
+) -> BoxStream<'static, Result<T, LlmClientError>> {
+ let input = input.clone().unbind();
+ futures::stream::unfold(Some(input), move |state| async move {
+ let input = state?;
+ let next = Python::attach(|py| {
+ let locals = pyo3_async_runtimes::tokio::get_current_locals(py)?;
+ pyo3_async_runtimes::into_future_with_locals(
+ &locals,
+ input.bind(py).call_method0("_read")?,
+ )
+ });
+ let result = match next {
+ Ok(next) => next.await,
+ Err(error) => Err(error),
+ };
+ match result {
+ Ok(item) => {
+ let item = Python::attach(|py| convert(item.bind(py))).map_err(|error| {
+ LlmClientError::Ffi {
+ source: Box::new(error),
+ }
+ });
+ Some((item, Some(input)))
+ }
+ Err(error) if Python::attach(|py| error.is_instance_of::<PyStopAsyncIteration>(py)) => {
+ None
+ }
+ Err(error) => Some((
+ Err(LlmClientError::Ffi {
+ source: Box::new(error),
+ }),
+ None,
+ )),
+ }
+ })
+ .boxed()
+}
+
+fn output_stream<T: Serialize + Send + 'static>(
+ module: &Bound<'_, PyModule>,
+ input: Bound<'_, PyAny>,
+ stream: BoxStream<'static, Result<T, LlmStreamError>>,
+) -> PyResult<Py<PyAny>> {
+ let stream = stream.map(|item| {
+ Python::attach(|py| match item {
+ Ok(event) => to_python(py, &event),
+ Err(LlmStreamError::Client(error)) => Err(client_error(py, error)),
+ Err(LlmStreamError::Upstream(error)) => Err(translation_error(
+ py,
+ "UpstreamFailure",
+ format!("upstream stream error: {error}"),
+ json!({"error": error}),
+ )),
+ })
+ });
+ let native = Py::new(
+ module.py(),
+ NativeStream {
+ inner: Arc::new(Mutex::new(stream.boxed())),
+ },
+ )?;
+ module
+ .getattr("_TranslationStream")?
+ .call1((input, native))
+ .map(Bound::unbind)
+}
+
+fn translation_error(py: Python<'_>, kind: &str, message: String, details: Value) -> PyErr {
+ let error = TranslationError::new_err(message);
+ let attach = || -> PyResult<()> {
+ let value = error.value(py);
+ value.setattr("kind", kind)?;
+ value.setattr("details", to_python(py, &details)?)
+ };
+ match attach() {
+ Ok(()) => error,
+ Err(error) => error,
+ }
+}
+
+fn client_error(py: Python<'_>, error: LlmClientError) -> PyErr {
+ if let LlmClientError::Ffi { source } = &error
+ && let Some(error) = source.downcast_ref::<PyErr>()
+ {
+ return error.clone_ref(py);
+ }
+ let (kind, details) = match &error {
+ LlmClientError::ResponseTranslation(message) => {
+ ("ResponseTranslation", json!({"message": message}))
+ }
+ LlmClientError::Configuration { message } => ("Configuration", json!({"message": message})),
+ LlmClientError::InvalidRequest { message } => {
+ ("InvalidRequest", json!({"message": message}))
+ }
+ LlmClientError::RequestTranslation(message) => {
+ ("RequestTranslation", json!({"message": message}))
+ }
+ LlmClientError::RequestEncoding(message) => {
+ ("RequestEncoding", json!({"message": message}))
+ }
+ LlmClientError::ResponseStateLimitExceeded { limit } => {
+ ("ResponseStateLimitExceeded", json!({"limit": limit}))
+ }
+ LlmClientError::ResponseStateConflict => ("ResponseStateConflict", json!({})),
+ LlmClientError::TemporarilyUnavailable => ("TemporarilyUnavailable", json!({})),
+ LlmClientError::Transport { source } => {
+ ("Transport", json!({"message": source.to_string()}))
+ }
+ LlmClientError::Timeout { source } => ("Timeout", json!({"message": source.to_string()})),
+ LlmClientError::ContextWindowExceeded { model, message } => (
+ "ContextWindowExceeded",
+ json!({"model": model, "message": message}),
+ ),
+ LlmClientError::UpstreamHttp { status, body } => (
+ "UpstreamHttp",
+ json!({"status": status.as_u16(), "body": body}),
+ ),
+ LlmClientError::InvalidResponse { source } => {
+ ("InvalidResponse", json!({"message": source.to_string()}))
+ }
+ LlmClientError::Host { source } => ("Host", json!({"message": source.to_string()})),
+ LlmClientError::Ffi { source } => ("Ffi", json!({"message": source.to_string()})),
+ _ => ("Other", json!({"message": error.to_string()})),
+ };
+ translation_error(py, kind, error.to_string(), details)
}
fn translation_output(
@@ -160,17 +407,9 @@ fn translation_output(
}
fn into_python_result<T>(py: Python<'_>, result: Result<T, CoreTranslationError>) -> PyResult<T> {
- match result {
- Ok(output) => Ok(output),
- Err(error) => {
- let exception = TranslationError::new_err(error.to_string());
- exception.value(py).setattr("kind", error.kind())?;
- exception
- .value(py)
- .setattr("details", to_python(py, &error_details(&error))?)?;
- Err(exception)
- }
- }
+ result.map_err(|error| {
+ translation_error(py, error.kind(), error.to_string(), error_details(&error))
+ })
}
fn error_details(error: &CoreTranslationError) -> Value {
diff --git a/switchyard/translation.py b/switchyard/translation.py
index 9bf480e50..789a5bcd2 100644
--- a/switchyard/translation.py
+++ b/switchyard/translation.py
@@ -1,7 +1,7 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-"""Translate buffered provider requests and responses.
+"""Translate provider requests, responses, and streams.
from switchyard.translation import TranslationEngine, WireFormat
@@ -16,7 +16,7 @@
All methods accept a WireFormat member or its string value for each format.
Bodies and IR values must be dictionaries containing JSON-compatible values.
-Translate and encode results have "body" and "diagnostics" keys. decode_request
+Buffered translate and encode results have "body" and "diagnostics" keys. decode_request
returns "request" and "diagnostics" keys; decode_response returns "response" and
"diagnostics" keys. Diagnostics contain severity (Info,
Warning, or Error), code, message, source, target, and path; the last three fields
@@ -44,6 +44,47 @@ decoded request's extensions when encoding the response::
request_extensions uses the IR shape {"fields": {...}}. Omitting it, passing None,
or passing an empty dictionary or {"fields": {}} all encode without request context.
+
+Use translate_stream to convert SSE byte chunks into provider event dictionaries::
+
+ async with engine.translate_stream(
+ backend_format,
+ source_format,
+ byte_stream,
+ served_model="selected-model",
+ request_extensions=decoded_request["request"]["extensions"],
+ ) as encoded:
+ async for event in encoded:
+ handle_provider_event(event)
+
+Use decode_stream and encode_stream separately when you need access to the IR
+events, such as to inspect or modify them between decoding and encoding::
+
+ async with engine.decode_stream(backend_format, byte_stream) as decoded:
+ async with engine.encode_stream(
+ source_format,
+ decoded,
+ served_model="selected-model",
+ request_extensions=decoded_request["request"]["extensions"],
+ ) as encoded:
+ async for event in encoded:
+ handle_provider_event(event)
+
+The stream methods return async iterators that also support async context management.
+Prefer async with to release translation resources on early exit. Each iterator owns
+its translation state and can outlive the engine. The caller owns the input stream
+and its transport; closing a translation iterator never calls the input's close
+methods. Cancelling a read ends that translation iterator and cancels its pending
+input read, which may also terminate the input iterator. Cleanup errors propagate.
+
+Decoded items are IR dictionaries with preservation and normalized keys. Encoded
+items are provider event dictionaries; the caller supplies any SSE output framing.
+Rust handles normal stream finalization. Closing early does not emit completion
+events. There is no separate diagnostics list for streaming: DecodeError and
+StreamError remain IR chunks, while Rust stream failures raise TranslationError.
+Python input exceptions retain their original types. Streaming TranslationError
+kinds also include client categories such as ResponseTranslation and Configuration;
+UpstreamFailure preserves the encoded error payload in details["error"].
"""
from switchyard_rust.translation import TranslationEngine, TranslationError, WireFormat
diff --git a/switchyard_rust/_translation_stream.py b/switchyard_rust/_translation_stream.py
new file mode 100644
index 000000000..bbedb1486
--- /dev/null
+++ b/switchyard_rust/_translation_stream.py
@@ -0,0 +1,93 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Lifecycle of a translation stream and the input reads it initiates."""
+
+import asyncio
+from collections.abc import AsyncIterable, AsyncIterator
+from types import TracebackType
+from typing import Any
+
+
+class _StreamInput:
+ def __init__(self, source: AsyncIterable[Any]) -> None:
+ self._iterator: AsyncIterator[Any] = aiter(source)
+ self._task: asyncio.Future[Any] | None = None
+ self._closed = False
+
+ async def _read(self) -> Any:
+ # Rust may have scheduled this coroutine just before aclose() started.
+ if self._closed:
+ raise asyncio.CancelledError
+ self._task = asyncio.ensure_future(anext(self._iterator))
+ try:
+ return await self._task
+ finally:
+ self._task = None
+
+ def cancel_read(self) -> None:
+ if self._closed:
+ return
+ self._closed = True
+ if self._task is not None:
+ self._task.cancel()
+
+
+class _TranslationStream:
+ def __init__(self, input_stream: _StreamInput, native: Any) -> None:
+ self._input: _StreamInput | None = input_stream
+ self._native: Any = native
+ self._next_future: asyncio.Future[dict[str, Any]] | None = None
+ self._closed = False
+
+ def __aiter__(self) -> "_TranslationStream":
+ return self
+
+ async def __anext__(self) -> dict[str, Any]:
+ if self._closed:
+ raise StopAsyncIteration
+ if self._next_future is not None:
+ raise RuntimeError("translation stream already has a pending read")
+ pending: asyncio.Future[dict[str, Any]] = self._native.__anext__()
+ self._next_future = pending
+ try:
+ # Keep the Rust poll alive until input cancellation and cleanup have completed.
+ event = await asyncio.shield(pending)
+ if self._closed:
+ raise StopAsyncIteration
+ return event
+ except BaseException:
+ await self.aclose()
+ raise
+ finally:
+ self._next_future = None
+
+ async def aclose(self) -> None:
+ self._closed = True
+ if self._input is not None:
+ self._input.cancel_read()
+ try:
+ pending = self._next_future
+ if pending is not None and not pending.done():
+ try:
+ await asyncio.shield(pending)
+ except asyncio.CancelledError:
+ # Only consume cancellation of the input read, not cancellation of aclose().
+ if not pending.done():
+ raise
+ except StopAsyncIteration:
+ pass
+ finally:
+ self._native = None
+ self._input = None
+
+ async def __aenter__(self) -> "_TranslationStream":
+ return self
+
+ async def __aexit__(
+ self,
+ exc_type: type[BaseException] | None,
+ exc_value: BaseException | None,
+ traceback: TracebackType | None,
+ ) -> None:
+ await self.aclose()
diff --git a/switchyard_rust/translation.py b/switchyard_rust/translation.py
index 35ca80d03..298aa4ca8 100644
--- a/switchyard_rust/translation.py
+++ b/switchyard_rust/translation.py
@@ -1,7 +1,7 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-"""Native bindings for buffered request and response translation."""
+"""Native bindings for request, response, and stream translation."""
from enum import Enum
from typing import TYPE_CHECKING, Any, final
@@ -18,6 +18,9 @@ class WireFormat(str, Enum):
if TYPE_CHECKING:
+ from collections.abc import AsyncIterable
+
+ from switchyard_rust._translation_stream import _TranslationStream
class TranslationError(Exception):
"""Translation failure with a readable message and structured details."""
@@ -27,7 +30,7 @@ if TYPE_CHECKING:
@final
class TranslationEngine:
- """Translate buffered provider JSON using the default translation policy.
+ """Translate provider JSON and streams.
Calls are independent. Request context is not retained for responses.
"""
@@ -74,6 +77,35 @@ if TYPE_CHECKING:
"""Translate a complete response, returning body and diagnostics."""
...
+ def decode_stream(
+ self, source: WireFormat | str, byte_stream: AsyncIterable[bytes]
+ ) -> _TranslationStream:
+ """Decode SSE bytes into IR events; prefer consumption inside async with."""
+ ...
+
+ def encode_stream(
+ self,
+ target: WireFormat | str,
+ chunks: AsyncIterable[dict[str, Any]],
+ *,
+ served_model: str | None = None,
+ request_extensions: dict[str, Any] | None = None,
+ ) -> _TranslationStream:
+ """Encode IR events into provider dictionaries; the caller owns input cleanup."""
+ ...
+
+ def translate_stream(
+ self,
+ source: WireFormat | str,
+ target: WireFormat | str,
+ byte_stream: AsyncIterable[bytes],
+ *,
+ served_model: str | None = None,
+ request_extensions: dict[str, Any] | None = None,
+ ) -> _TranslationStream:
+ """Translate SSE byte chunks into target-format event dictionaries."""
+ ...
+
def __getattr__(name: str) -> object:
if name in {"TranslationEngine", "TranslationError"}:
diff --git a/tests/test_translation_stream.py b/tests/test_translation_stream.py
new file mode 100644
index 000000000..e556ec6ac
--- /dev/null
+++ b/tests/test_translation_stream.py
@@ -0,0 +1,335 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Streaming translation, error propagation, and caller-owned input lifetimes."""
+
+import asyncio
+import json
+import weakref
+
+import pytest
+
+from switchyard.libsy import (
+ CustomClassifierConfig,
+ LlmClassifierConfig,
+ LlmResponse,
+ Step,
+ algorithms,
+)
+from switchyard.translation import TranslationEngine, TranslationError, WireFormat
+
+
+async def byte_stream(events, *, terminal=True):
+ payload = b"".join(
+ ("data: " + json.dumps(event, ensure_ascii=False) + "\n\n").encode() for event in events
+ )
+ if terminal:
+ payload += b"data: [DONE]\n\n"
+ # Split SSE boundaries and multi-byte UTF-8 characters across reads.
+ for byte in payload:
+ yield bytes([byte])
+
+
+async def items(values):
+ for value in values:
+ yield value
+
+
+async def test_stream_translation_preserves_ir_and_restores_extensions():
+ engine = TranslationEngine()
+ request = engine.decode_request(
+ WireFormat.OPENAI_RESPONSES,
+ {
+ "input": "Search",
+ "tools": [
+ {
+ "type": "namespace",
+ "name": "mcp__docs",
+ "tools": [{"type": "function", "name": "search", "parameters": {}}],
+ }
+ ],
+ },
+ )
+ events = [
+ {
+ "id": "response-1",
+ "model": "backend",
+ "choices": [
+ {
+ "index": 0,
+ "delta": {"role": "assistant", "content": "Héllo"},
+ "finish_reason": None,
+ }
+ ],
+ },
+ {
+ "choices": [
+ {
+ "index": 0,
+ "delta": {
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "call-1",
+ "type": "function",
+ "function": {
+ "name": "mcp__docs__search",
+ "arguments": '{"query":"streaming"}',
+ },
+ }
+ ]
+ },
+ "finish_reason": "tool_calls",
+ }
+ ]
+ },
+ ]
+ async with engine.decode_stream(WireFormat.OPENAI_CHAT, byte_stream(events)) as decoded:
+ ir = [event async for event in decoded]
+ assert [event["preservation"]["raw"] for event in ir] == events
+ assert {"TextDelta": {"index": 0, "text": "Héllo"}} in ir[0]["normalized"]
+
+ # Direct iteration works, including replay and empty extension equivalence.
+ for extensions in (None, {}, {"fields": {}}):
+ replay = engine.encode_stream("openai_chat", items(ir), request_extensions=extensions)
+ assert [event async for event in replay] == events
+
+ encoded = engine.encode_stream(
+ WireFormat.OPENAI_RESPONSES,
+ items(ir),
+ served_model="selected",
+ request_extensions=request["request"]["extensions"],
+ )
+ del engine
+ async with encoded:
+ output = [event async for event in encoded]
+ assert output[-1]["type"] == "response.completed"
+ assert output[-1]["response"]["model"] == "selected"
+ call = next(
+ item for item in output[-1]["response"]["output"] if item["type"] == "function_call"
+ )
+ assert call["name"] == "search"
+ assert call["namespace"] == "mcp__docs"
+
+ engine = TranslationEngine()
+ async with engine.translate_stream(
+ "openai_chat",
+ "openai_responses",
+ byte_stream(events),
+ served_model="selected",
+ request_extensions=request["request"]["extensions"],
+ ) as translated:
+ translated_output = [event async for event in translated]
+ assert translated_output[-1]["response"]["model"] == "selected"
+ assert translated_output[-1]["response"]["output"] == output[-1]["response"]["output"]
+
+ async with engine.decode_stream("openai_chat", byte_stream(events)) as decoded:
+ async with engine.encode_stream("anthropic_messages", decoded) as encoded:
+ output = [event async for event in encoded]
+ assert output[-1]["type"] == "message_stop"
+ assert any(event.get("delta", {}).get("text") == "Héllo" for event in output)
+ translated = engine.translate_stream("openai_chat", "anthropic_messages", byte_stream(events))
+ assert [event async for event in translated] == output
+
+ # A decoded stream can also be consumed by the existing libsy response adapter.
+ classifier = algorithms.llm_classifier(
+ LlmClassifierConfig.custom(
+ default_target="capable",
+ config=CustomClassifierConfig(
+ "Choose a target.",
+ {
+ "type": "object",
+ "properties": {"target": {"type": "string"}},
+ "required": ["target"],
+ },
+ "/target",
+ ),
+ )
+ )
+ classification = [
+ {"choices": [{"delta": {"content": '{"target":"efficient"}'}, "finish_reason": "stop"}]}
+ ]
+ async with engine.decode_stream("openai_chat", byte_stream(classification)) as decoded:
+
+ async def response():
+ return LlmResponse.Stream(decoded)
+
+ outcome = None
+ async for step in classifier.run_stream(
+ request["request"],
+ {
+ "judge": ["judge"],
+ "capable": ["large"],
+ "efficient": ["small"],
+ "any": ["large", "small"],
+ },
+ ):
+ if isinstance(step, Step.CallModel):
+ await step.call.respond(response())
+ elif isinstance(step, Step.Done):
+ outcome = step.outcome
+ assert outcome is not None
+ assert outcome.selected_model_ids[0] == "small"
+
+
+ "method,kind",
+ [
+ ("decode_stream", "ResponseTranslation"),
+ ("encode_stream", "Configuration"),
+ ("translate_stream", "ResponseTranslation"),
+ ],
+)
+async def test_stream_errors_preserve_exceptions(method, kind):
+ engine = TranslationEngine()
+ make_stream = getattr(engine, method)
+ formats = (
+ ("openai_chat", "anthropic_messages") if method == "translate_stream" else ("openai_chat",)
+ )
+ with pytest.raises(TranslationError, match="no stream codec registered for fake") as caught:
+ make_stream("fake", *formats[1:], items([]))
+ assert caught.value.kind == kind
+ assert caught.value.details == {"message": "no stream codec registered for fake"}
+
+ if method == "translate_stream":
+ with pytest.raises(TranslationError, match="no stream codec registered for fake") as caught:
+ make_stream("openai_chat", "fake", items([]))
+ assert caught.value.kind == "Configuration"
+
+ if method != "decode_stream":
+ with pytest.raises(ValueError):
+ make_stream(*formats, items([]), request_extensions={"fields": []})
+
+ failure = OSError("input failed")
+
+ async def failing_input():
+ raise failure
+ yield # pragma: no cover
+
+ async with make_stream(*formats, failing_input()) as stream:
+ with pytest.raises(OSError) as caught:
+ await anext(stream)
+ assert caught.value is failure
+
+ invalid = [] if method == "encode_stream" else "not bytes"
+ async with make_stream(*formats, items([invalid])) as stream:
+ with pytest.raises(TypeError):
+ await anext(stream)
+
+ if method == "encode_stream":
+ async with engine.encode_stream(
+ "openai_chat", items([{"normalized": "invalid"}])
+ ) as stream:
+ with pytest.raises(ValueError):
+ await anext(stream)
+ return
+
+ for payload in (b"data: {\n\n", b""):
+ async with make_stream(*formats, items([payload])) as stream:
+ with pytest.raises(TranslationError) as caught:
+ await anext(stream)
+ assert caught.value.kind == "ResponseTranslation"
+ assert caught.value.details["message"]
+
+
[email protected]("method", ["decode_encode", "translate_stream"])
+async def test_stream_error_events_become_upstream_failures(method):
+ engine = TranslationEngine()
+ for raw, variant, message in [
+ ([], "DecodeError", "OpenAI stream event is not an object"),
+ ({"error": {"message": "provider failed"}}, "StreamError", "provider failed"),
+ ]:
+ if method == "decode_encode":
+ async with engine.decode_stream(
+ "openai_chat", byte_stream([raw], terminal=False)
+ ) as decoded:
+ ir = [event async for event in decoded]
+ assert ir == [
+ {
+ "preservation": {"source": "openai_chat", "raw": raw},
+ "normalized": [{variant: {"message": message}}],
+ }
+ ]
+ stream = engine.encode_stream("anthropic_messages", items(ir))
+ else:
+ stream = engine.translate_stream(
+ "openai_chat", "anthropic_messages", byte_stream([raw], terminal=False)
+ )
+ async with stream:
+ with pytest.raises(TranslationError) as caught:
+ async for _ in stream:
+ pass
+ assert caught.value.kind == "UpstreamFailure"
+ assert caught.value.details["error"]["type"] == "error"
+ assert caught.value.details["error"]["error"]["message"] == message
+
+
[email protected]("method", ["decode_stream", "encode_stream", "translate_stream"])
[email protected]("exit_mode", ["break", "cancel", "close", "cleanup-error"])
+async def test_stream_cleanup_owns_only_translation_resources(method, exit_mode):
+ started = asyncio.Event()
+ cleaned = asyncio.Event()
+ cleanup_error = RuntimeError("input read cleanup failed")
+
+ class Input:
+ def __init__(self):
+ self.reads = 0
+ self.close_calls = 0
+
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ self.reads += 1
+ if self.reads == 1:
+ if method != "encode_stream":
+ return b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n'
+ return {
+ "preservation": None,
+ "normalized": [{"TextDelta": {"index": 0, "text": "hello"}}],
+ }
+ started.set()
+ try:
+ await asyncio.Future()
+ finally:
+ await asyncio.sleep(0)
+ cleaned.set()
+ if exit_mode == "cleanup-error":
+ raise cleanup_error
+
+ async def aclose(self):
+ self.close_calls += 1
+
+ source = Input()
+ source_ref = weakref.ref(source)
+ engine = TranslationEngine()
+ if method == "translate_stream":
+ stream = engine.translate_stream("openai_chat", "openai_chat", source)
+ else:
+ stream = getattr(engine, method)("openai_chat", source)
+ del engine
+ async with stream:
+ async for _ in stream:
+ break
+ assert source.reads == 1
+ if exit_mode != "break":
+ pending = asyncio.create_task(anext(stream))
+ await asyncio.wait_for(started.wait(), 2)
+ if exit_mode == "close":
+ await asyncio.gather(stream.aclose(), stream.aclose())
+ else:
+ pending.cancel()
+ expected = RuntimeError if exit_mode == "cleanup-error" else asyncio.CancelledError
+ with pytest.raises(expected) as caught:
+ await asyncio.wait_for(pending, 2)
+ if exit_mode == "cleanup-error":
+ assert caught.value is cleanup_error
+ assert isinstance(caught.value.__context__, asyncio.CancelledError)
+ assert cleaned.is_set()
+ await stream.aclose()
+ with pytest.raises(StopAsyncIteration):
+ await anext(stream)
+ assert source.close_calls == 0
+ if exit_mode == "break":
+ del source
+ assert source_ref() is None