Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! Minimal Python API for running Rust-owned libsy algorithms.
use std::collections::{BTreeMap, HashMap};
use std::sync::Arc;
use futures::StreamExt;
use http::HeaderMap;
use http::header::{HeaderName, HeaderValue};
use pyo3::exceptions::{PyBaseException, PyStopAsyncIteration, PyTypeError, PyValueError};
use pyo3::prelude::*;
use pyo3::types::PyDict;
use serde_json::Value;
use switchyard_libsy::{
Algorithm, CallDecision, CallModel, CapabilityJudgeConfig, ClassifierContractConfig,
ClassifierResponseFormat, ClassifyTrigger, CustomClassifierConfig, CustomClassifierPolicy,
DEFAULT_UNMATCHED_STEPS, DecisionJudgeConfig, DeescalationConfig, EscalationJudgeConfig,
HandoffNoteConfig, LibsyError as RustLibsyError, LlmCapabilityConfig, LlmClassifierConfig,
LlmFallback, LlmTaskClassifier, Noop, PickerMode, Random, RoutingOutcome, RuntimeModels,
StageRouter, StageRouterConfig, State, Step as RustStep, StepStream, TaskClassifierConfig,
ToolSemantics,
};
use switchyard_protocol::{
Category, DecisionResponse, LlmClientError, LlmResponse, LlmResponseStream,
LlmResponseStreamEvent, Metadata, ModelId, Request, Response,
};
use switchyard_runner::{AlgorithmFactory, require_target_labels};
use tokio::sync::Mutex;
use crate::errors::{ContextWindowExceededError, py_libsy_error};
use crate::py_serde::{from_python, to_python};
/// The Python API keeps its `session_affinity` flag, which selects the per-session trigger.
fn classify_trigger(session_affinity: bool) -> ClassifyTrigger {
if session_affinity {
ClassifyTrigger::NewSession
} else {
ClassifyTrigger::EveryRequest
}
}
fn parse_category(value: &str) -> PyResult<Category> {
value.parse().map_err(PyValueError::new_err)
}
/// Convert one scope's `{group: [model_id]}` mapping into its typed form.
///
/// `any`, `capable`, `efficient`, and `judge` are the groups algorithms reason
/// about. Any other key is a deployment-defined group, selectable by name only
/// by a custom classifier's policy.
fn category_models_from_python(
models: HashMap<String, Vec<String>>,
) -> PyResult<HashMap<Category, Vec<ModelId>>> {
models
.into_iter()
.map(|(category, ids)| {
Ok((
parse_category(&category)?,
ids.into_iter().map(ModelId::from).collect(),
))
})
.collect()
}
/// Convert Python-owned headers into the request metadata expected by libsy.
fn header_map_from_python(headers: &HashMap<String, String>) -> PyResult<http::HeaderMap> {
let mut result = http::HeaderMap::new();
for (name, value) in headers {
let name = HeaderName::from_bytes(name.as_bytes())
.map_err(|error| PyValueError::new_err(error.to_string()))?;
let value = HeaderValue::from_str(value)
.map_err(|error| PyValueError::new_err(error.to_string()))?;
result
.try_append(name, value)
.map_err(|error| PyValueError::new_err(error.to_string()))?;
}
Ok(result)
}
/// Attach normalized host metadata to a provider-neutral request.
fn request_from_python(
request: &Bound<'_, PyAny>,
headers: Option<HashMap<String, String>>,
) -> PyResult<Request> {
let headers = headers.as_ref().map(header_map_from_python).transpose()?;
let mut llm_request = from_python(request)?;
switchyard_translation::normalize_compaction(&mut llm_request, headers.as_ref());
Ok(Request {
llm_request,
raw_request: None,
metadata: headers.map(|headers| Metadata::from_headers(&headers)),
})
}
/// Classifier settings shared by standalone and stage-router classifiers.
#[pyclass(
name = "TaskClassifierConfig",
module = "switchyard.libsy",
frozen,
skip_from_py_object
)]
#[derive(Clone)]
struct PyTaskClassifierConfig {
inner: TaskClassifierConfig,
}
impl PyTaskClassifierConfig {
fn clone_core(&self) -> TaskClassifierConfig {
self.inner.clone()
}
}
/// Settings for returning an escalated session to the efficient tier.
///
/// `strong_min_calls` and `confirmations` must be positive; `strong_max_calls`, when set,
/// must not be lower than `strong_min_calls`. Classifier construction reports invalid values
/// as `ValueError`.
#[pyclass(
name = "DeescalationConfig",
module = "switchyard.libsy",
frozen,
skip_from_py_object
)]
#[derive(Clone)]
struct PyDeescalationConfig {
inner: DeescalationConfig,
}
#[pymethods]
impl PyDeescalationConfig {
#[new]
#[pyo3(signature = (
*,
strong_min_calls,
confirmations,
strong_max_calls=None,
weak_cooldown_calls=0
))]
fn new(
strong_min_calls: u32,
confirmations: u32,
strong_max_calls: Option<u32>,
weak_cooldown_calls: u32,
) -> Self {
Self {
inner: DeescalationConfig {
strong_min_calls,
strong_max_calls,
confirmations,
weak_cooldown_calls,
},
}
}
}
/// Settings for response-based escalation classification.
#[pyclass(
name = "EscalationClassifierConfig",
module = "switchyard.libsy",
frozen,
skip_from_py_object
)]
#[derive(Clone)]
struct PyEscalationClassifierConfig {
contract: ClassifierContractConfig,
judge: EscalationJudgeConfig,
max_output_tokens: u64,
judge_deadline_ms: Option<u64>,
}
#[pymethods]
impl PyEscalationClassifierConfig {
#[new]
#[pyo3(signature = (
*,
confirmations=2,
recent_turn_window=28,
window_message_chars=500,
deescalation=None,
max_output_tokens=4096,
judge_deadline_ms=None,
prompt=None,
response_format_type="json_schema"
))]
#[allow(clippy::too_many_arguments)]
fn new(
py: Python<'_>,
confirmations: u32,
recent_turn_window: usize,
window_message_chars: usize,
deescalation: Option<Py<PyDeescalationConfig>>,
max_output_tokens: u64,
judge_deadline_ms: Option<u64>,
prompt: Option<String>,
response_format_type: &str,
) -> PyResult<Self> {
Ok(Self {
contract: classifier_contract(prompt, response_format_type)?,
judge: EscalationJudgeConfig {
confirmations,
recent_turn_window,
window_message_chars,
deescalation: deescalation
.map(|config| config.bind(py).try_borrow().map(|config| config.inner))
.transpose()?,
},
max_output_tokens,
judge_deadline_ms,
})
}
}
/// Settings for a classifier with a user-supplied verdict schema.
#[pyclass(
name = "CustomClassifierConfig",
module = "switchyard.libsy",
frozen,
skip_from_py_object
)]
#[derive(Clone)]
struct PyCustomClassifierConfig {
inner: CustomClassifierConfig,
}
impl PyCustomClassifierConfig {
fn clone_core(&self) -> CustomClassifierConfig {
self.inner.clone()
}
}
#[pymethods]
impl PyCustomClassifierConfig {
#[new]
#[pyo3(signature = (
prompt,
response_schema,
selector,
*,
session_affinity=false,
message_hash_fallback=false,
recent_turn_window=None,
max_output_tokens=4096,
judge_deadline_ms=None
))]
#[allow(clippy::too_many_arguments)]
fn new(
prompt: String,
response_schema: &Bound<'_, PyAny>,
selector: String,
session_affinity: bool,
message_hash_fallback: bool,
recent_turn_window: Option<usize>,
max_output_tokens: u64,
judge_deadline_ms: Option<u64>,
) -> PyResult<Self> {
// Convert the Python schema into serde JSON and pair it with the target-selector policy;
// conversion failures propagate to Python through `PyResult`.
let mut inner = CustomClassifierConfig::new(
prompt,
from_python::<Value>(response_schema)?,
CustomClassifierPolicy::target_selector(selector),
);
inner.classify_trigger = classify_trigger(session_affinity);
inner.message_hash_fallback = message_hash_fallback;
inner.recent_turn_window = recent_turn_window;
inner.max_output_tokens = max_output_tokens;
inner.judge_deadline_ms = judge_deadline_ms;
Ok(Self { inner })
}
}
/// Construction settings for a Python-hosted LLM classifier.
#[pyclass(
name = "LlmClassifierConfig",
module = "switchyard.libsy",
frozen,
skip_from_py_object
)]
struct PyLlmClassifierConfig {
inner: LlmClassifierConfig,
}
#[pymethods]
impl PyLlmClassifierConfig {
/// Configure capability routing between efficient and capable targets.
#[staticmethod]
#[pyo3(signature = (*, config))]
fn capability(py: Python<'_>, config: Py<PyTaskClassifierConfig>) -> PyResult<Self> {
Ok(Self {
inner: LlmClassifierConfig::Capability {
config: config.bind(py).try_borrow()?.clone_core(),
},
})
}
/// Configure response-based escalation between efficient and capable targets.
#[staticmethod]
#[pyo3(signature = (*, config))]
fn escalation(py: Python<'_>, config: Py<PyEscalationClassifierConfig>) -> PyResult<Self> {
let config = config.bind(py).try_borrow()?;
Ok(Self {
inner: LlmClassifierConfig::Escalation {
contract: config.contract.clone(),
config: config.judge.clone(),
max_output_tokens: config.max_output_tokens,
judge_deadline_ms: config.judge_deadline_ms,
},
})
}
/// Configure schema-driven routing across runtime model categories.
#[staticmethod]
#[pyo3(signature = (*, default_target, config))]
fn custom(
py: Python<'_>,
default_target: String,
config: Py<PyCustomClassifierConfig>,
) -> PyResult<Self> {
let config = config.bind(py).try_borrow()?.clone_core();
Ok(Self {
inner: LlmClassifierConfig::Custom {
default_target: parse_category(&default_target)?,
config,
},
})
}
}
#[pymethods]
impl PyTaskClassifierConfig {
/// Route capable when a decision model's choice probability exceeds a cutoff.
#[staticmethod]
#[pyo3(signature = (*, cutoff=None, instructions=None, choices=None, apply_cutoff_to=None))]
fn decision(
cutoff: Option<f64>,
instructions: Option<String>,
choices: Option<&Bound<'_, PyAny>>,
apply_cutoff_to: Option<String>,
) -> PyResult<Self> {
let defaults = DecisionJudgeConfig::default();
Ok(Self {
inner: TaskClassifierConfig {
judge: CapabilityJudgeConfig::Decision(DecisionJudgeConfig {
cutoff,
instructions,
choices: match choices {
Some(choices) => from_python(choices)?,
None => defaults.choices,
},
apply_cutoff_to: apply_cutoff_to.unwrap_or(defaults.apply_cutoff_to),
}),
..TaskClassifierConfig::default()
},
})
}
#[new]
#[pyo3(signature = (
base_threshold,
*,
threshold_step=0.0,
session_affinity=false,
message_hash_fallback=false,
recent_turn_window=None,
max_output_tokens=4096,
judge_deadline_ms=None,
prompt=None,
response_format_type="json_schema"
))]
#[allow(clippy::too_many_arguments)]
fn new(
base_threshold: f64,
threshold_step: f64,
session_affinity: bool,
message_hash_fallback: bool,
recent_turn_window: Option<usize>,
max_output_tokens: u64,
judge_deadline_ms: Option<u64>,
prompt: Option<String>,
response_format_type: &str,
) -> PyResult<Self> {
Ok(Self {
inner: TaskClassifierConfig {
judge: CapabilityJudgeConfig::Llm(LlmCapabilityConfig {
base_threshold,
threshold_step,
unmatched_steps: DEFAULT_UNMATCHED_STEPS,
contract: classifier_contract(prompt, response_format_type)?,
max_output_tokens,
judge_deadline_ms,
}),
fail_open: true,
classify_trigger: classify_trigger(session_affinity),
message_hash_fallback,
recent_turn_window,
},
})
}
}
fn classifier_contract(
prompt: Option<String>,
response_format_type: &str,
) -> PyResult<ClassifierContractConfig> {
let mut contract = ClassifierContractConfig::default();
if let Some(prompt) = prompt {
contract = contract.with_prompt(prompt);
}
let response_format_type = match response_format_type {
"json_schema" => ClassifierResponseFormat::JsonSchema,
"json_object" => ClassifierResponseFormat::JsonObject,
other => {
return Err(PyValueError::new_err(format!(
"response_format_type must be 'json_schema' or 'json_object', got {other:?}"
)));
}
};
Ok(contract.with_response_format_type(response_format_type))
}
/// Judge target and policy used when stage-router signals are inconclusive.
#[pyclass(
name = "LlmFallback",
module = "switchyard.libsy",
frozen,
skip_from_py_object
)]
struct PyLlmFallback {
config: Py<PyTaskClassifierConfig>,
}
impl PyLlmFallback {
fn clone_core(&self, py: Python<'_>) -> PyResult<LlmFallback> {
Ok(LlmFallback {
target: None,
config: self.config.bind(py).try_borrow()?.clone_core(),
})
}
}
#[pymethods]
impl PyLlmFallback {
#[new]
#[pyo3(signature = (*, config))]
fn new(config: Py<PyTaskClassifierConfig>) -> Self {
Self { config }
}
}
/// A normalized aggregate response or a live stream of normalized response events.
#[pyclass(name = "LlmResponse", module = "switchyard.libsy", frozen)]
enum PyLlmResponse {
/// A fully buffered normalized response dictionary.
#[pyo3(constructor = (response))]
Agg { response: Py<PyAny> },
/// An async iterator of normalized response-event dictionaries.
#[pyo3(constructor = (stream))]
Stream { stream: Py<PyAny> },
}
impl PyLlmResponse {
fn to_core(&self, py: Python<'_>, model: ModelId) -> PyResult<LlmResponse> {
match self {
Self::Agg { response } => from_python(response.bind(py)).map(LlmResponse::Agg),
Self::Stream { stream } => {
python_response_stream(py, stream.clone_ref(py), model).map(LlmResponse::Stream)
}
}
}
}
fn ffi_error(error: PyErr) -> LlmClientError {
LlmClientError::Ffi {
source: Box::new(error),
}
}
fn python_client_error(py: Python<'_>, error: PyErr, model: &ModelId) -> LlmClientError {
if error.is_instance_of::<ContextWindowExceededError>(py) {
LlmClientError::ContextWindowExceeded {
model: model.clone(),
message: error.value(py).to_string(),
}
} else {
ffi_error(error)
}
}
fn python_response_stream(
py: Python<'_>,
stream: Py<PyAny>,
model: ModelId,
) -> PyResult<LlmResponseStream> {
let iterator = stream.bind(py).call_method0("__aiter__")?.unbind();
// Rust polls on Tokio, so retain the Python task's event loop and context for every item.
let locals = pyo3_async_runtimes::tokio::get_current_locals(py)?;
let stream = futures::stream::unfold(Some((iterator, locals, model)), |state| async move {
let (iterator, locals, model) = state?;
let next = Python::attach(|py| {
pyo3_async_runtimes::into_future_with_locals(
&locals,
iterator.bind(py).call_method0("__anext__")?,
)
});
match next {
Ok(next) => match next.await {
Ok(item) => {
let event =
Python::attach(|py| from_python::<LlmResponseStreamEvent>(item.bind(py)))
.map_err(ffi_error);
Some((event, Some((iterator, locals, model))))
}
Err(error) => {
if Python::attach(|py| error.is_instance_of::<PyStopAsyncIteration>(py)) {
None
} else {
let error = Python::attach(|py| python_client_error(py, error, &model));
Some((Err(error), None))
}
}
},
Err(error) => Some((Err(ffi_error(error)), None)),
}
});
Ok(Box::pin(stream))
}
/// One model call yielded by [`PyAlgorithm::run_stream`].
#[pyclass(name = "ModelCall", module = "switchyard.libsy")]
struct PyModelCall {
inner: Option<CallModel>,
algorithm: String,
request: Py<PyAny>,
models: Vec<String>,
}
impl PyModelCall {
fn new(py: Python<'_>, call: CallModel) -> PyResult<Self> {
let request = to_python(py, &call.request.llm_request)?;
Ok(Self {
algorithm: call.algorithm.clone(),
models: call.models.iter().map(ToString::to_string).collect(),
inner: Some(call),
request,
})
}
fn take(&mut self) -> PyResult<CallModel> {
self.inner
.take()
.ok_or_else(|| py_libsy_error("model call has already been completed"))
}
}
#[pymethods]
impl PyModelCall {
/// The algorithm that produced this call.
#[getter]
fn algorithm(&self) -> &str {
&self.algorithm
}
/// The normalized LLM request to serve as a Python dictionary.
#[getter]
fn request(&self, py: Python<'_>) -> Py<PyAny> {
self.request.clone_ref(py)
}
/// Candidate models in the order the host should try them.
#[getter]
fn models(&self) -> Vec<String> {
self.models.clone()
}
/// Await a normalized response, cancelling Python work if the algorithm stops waiting.
fn respond<'py>(
slf: PyRef<'py, Self>,
py: Python<'py>,
work: &Bound<'py, PyAny>,
) -> PyResult<Bound<'py, PyAny>> {
py.import("switchyard_rust._libsy_async")?
.getattr("respond")?
.call1((slf, work))
}
fn _respond<'py>(
&mut self,
py: Python<'py>,
work: Bound<'py, PyAny>,
) -> PyResult<Bound<'py, PyAny>> {
let model = self
.inner
.as_ref()
.ok_or_else(|| py_libsy_error("model call has already been completed"))?
.request
.llm_request
.model
.as_ref()
.map(ModelId::new)
.ok_or_else(|| py_libsy_error("model call request is missing its selected model"))?;
let locals = pyo3_async_runtimes::tokio::get_current_locals(py)?;
let work = pyo3_async_runtimes::into_future_with_locals(&locals, work)?;
let call = self.take()?;
let metadata = call.request.metadata.clone();
pyo3_async_runtimes::tokio::future_into_py(py, async move {
call.respond(async move {
let result = work.await;
Python::attach(|py| {
let response = result.map_err(|error| {
RustLibsyError::client_call(
model.clone(),
python_client_error(py, error, &model),
)
})?;
let response = response
.bind(py)
.extract::<PyRef<'_, PyLlmResponse>>()
.map_err(|error| ffi_error(error.into()))
.map_err(|source| RustLibsyError::client_call(model.clone(), source))?;
let llm_response = response
.to_core(py, model.clone())
.map_err(ffi_error)
.map_err(|source| RustLibsyError::client_call(model, source))?;
Ok(Response {
llm_response,
metadata,
upstream_headers: HeaderMap::new(),
})
})
})
.await
.map_err(py_libsy_error)
})
}
/// Return a Python client failure through the same async reply path.
fn fail<'py>(
slf: PyRef<'py, Self>,
py: Python<'py>,
error: &Bound<'py, PyAny>,
) -> PyResult<Bound<'py, PyAny>> {
if !error.is_instance_of::<PyBaseException>() {
return Err(PyTypeError::new_err("error must derive from BaseException"));
}
py.import("switchyard_rust._libsy_async")?
.getattr("fail")?
.call1((slf, error))
}
}
/// A decision request owned by the Python host until it replies or drops the call.
#[pyclass(name = "DecisionCall", module = "switchyard.libsy")]
struct PyDecisionCall {
inner: Option<CallDecision>,
algorithm: String,
request: Py<PyAny>,
model: ModelId,
}
impl PyDecisionCall {
fn new(py: Python<'_>, call: CallDecision) -> PyResult<Self> {
let request = to_python(py, &call.request)?;
Ok(Self {
algorithm: call.algorithm.clone(),
model: call.model.clone(),
inner: Some(call),
request,
})
}
fn take(&mut self) -> PyResult<CallDecision> {
self.inner
.take()
.ok_or_else(|| py_libsy_error("decision call has already been completed"))
}
}
#[pymethods]
impl PyDecisionCall {
#[getter]
fn algorithm(&self) -> &str {
&self.algorithm
}
/// The provider-neutral decision request as a Python dictionary.
#[getter]
fn request(&self, py: Python<'_>) -> Py<PyAny> {
self.request.clone_ref(py)
}
#[getter]
fn model(&self) -> &str {
&self.model
}
/// Await a decision response, cancelling Python work if the algorithm stops waiting.
fn respond<'py>(
slf: PyRef<'py, Self>,
py: Python<'py>,
work: &Bound<'py, PyAny>,
) -> PyResult<Bound<'py, PyAny>> {
py.import("switchyard_rust._libsy_async")?
.getattr("respond")?
.call1((slf, work))
}
fn _respond<'py>(
&mut self,
py: Python<'py>,
work: Bound<'py, PyAny>,
) -> PyResult<Bound<'py, PyAny>> {
let locals = pyo3_async_runtimes::tokio::get_current_locals(py)?;
let work = pyo3_async_runtimes::into_future_with_locals(&locals, work)?;
let call = self.take()?;
let model = call.model.clone();
pyo3_async_runtimes::tokio::future_into_py(py, async move {
call.respond(async move {
let result = work.await;
Python::attach(|py| {
let response = result.map_err(|error| {
RustLibsyError::client_call(
model.clone(),
python_client_error(py, error, &model),
)
})?;
from_python::<DecisionResponse>(response.bind(py))
.map_err(ffi_error)
.map_err(|source| RustLibsyError::client_call(model, source))
})
})
.await
.map_err(py_libsy_error)
})
}
/// Return a Python client failure so the algorithm can apply its fallback policy.
fn fail<'py>(
slf: PyRef<'py, Self>,
py: Python<'py>,
error: &Bound<'py, PyAny>,
) -> PyResult<Bound<'py, PyAny>> {
if !error.is_instance_of::<PyBaseException>() {
return Err(PyTypeError::new_err("error must derive from BaseException"));
}
py.import("switchyard_rust._libsy_async")?
.getattr("fail")?
.call1((slf, error))
}
}
/// Identity and optional JSON evidence from the Rust routing outcome.
#[pyclass(name = "OutcomeMetadata", module = "switchyard.libsy", frozen)]
struct PyOutcomeMetadata {
inner: switchyard_libsy::OutcomeMetadata,
}
#[pymethods]
impl PyOutcomeMetadata {
/// UUIDv7 generated for this outcome.
#[getter]
fn outcome_id(&self) -> &str {
self.inner.outcome_id()
}
/// Name of the algorithm that produced this outcome.
#[getter]
fn algorithm(&self) -> &str {
&self.inner.algorithm
}
/// Optional evidence converted to ordinary Python JSON values.
#[getter]
fn evidence(&self, py: Python<'_>) -> PyResult<Option<Py<PyAny>>> {
self.inner
.evidence
.as_ref()
.map(|value| to_python(py, value))
.transpose()
}
}
/// The terminal routing selection, rewritten request, optional response, and metadata.
#[pyclass(name = "RoutingOutcome", module = "switchyard.libsy", frozen)]
struct PyRoutingOutcome {
selected_model_ids: Vec<String>,
request: Py<PyAny>,
fallback_request: Option<Py<PyAny>>,
#[pyo3(get)]
candidate_requests: Py<PyAny>,
response: Option<Py<PyAny>>,
#[pyo3(get)]
metadata: Option<Py<PyOutcomeMetadata>>,
}
#[pymethods]
impl PyRoutingOutcome {
/// Models selected by the algorithm, ordered best model first.
#[getter]
fn selected_model_ids(&self) -> Vec<String> {
self.selected_model_ids.clone()
}
/// The normalized request after routing-time rewrites.
#[getter]
fn request(&self, py: Python<'_>) -> Py<PyAny> {
self.request.clone_ref(py)
}
/// The request for the fallback candidates, when it differs from `request`.
#[getter]
fn fallback_request(&self, py: Python<'_>) -> Option<Py<PyAny>> {
self.fallback_request
.as_ref()
.map(|request| request.clone_ref(py))
}
/// An aggregate or streamed answer produced while routing, when one already exists.
#[getter]
fn response(&self, py: Python<'_>) -> Option<Py<PyAny>> {
self.response
.as_ref()
.map(|response| response.clone_ref(py))
}
}
/// Async Python iterator over one normalized Rust response stream.
#[pyclass(name = "_LlmResponseStream", module = "switchyard.libsy", frozen)]
struct PyLlmResponseStream {
inner: Arc<Mutex<LlmResponseStream>>,
}
#[pymethods]
impl PyLlmResponseStream {
fn __aiter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> {
slf
}
fn __anext__<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
let stream = Arc::clone(&self.inner);
pyo3_async_runtimes::tokio::future_into_py(py, async move {
match stream.lock().await.next().await {
Some(Ok(event)) => Python::attach(|py| to_python(py, &event)),
Some(Err(error)) => Err(py_libsy_error(error)),
None => Err(PyStopAsyncIteration::new_err(())),
}
})
}
}
fn response_to_python(py: Python<'_>, response: LlmResponse) -> PyResult<Py<PyAny>> {
let response = match response {
LlmResponse::Agg(response) => PyLlmResponse::Agg {
response: to_python(py, &response)?,
},
LlmResponse::Stream(stream) => PyLlmResponse::Stream {
stream: Py::new(
py,
PyLlmResponseStream {
inner: Arc::new(Mutex::new(stream)),
},
)?
.into_any(),
},
};
response
.into_pyobject(py)
.map(|response| response.unbind().into_any())
}
/// One item yielded by a Python algorithm stream.
#[pyclass(name = "Step", module = "switchyard.libsy", frozen)]
enum PyStep {
/// The host must serve the model call before the algorithm can continue.
CallModel { call: Py<PyModelCall> },
/// The host must serve the decision call before the algorithm can continue.
CallDecision { call: Py<PyDecisionCall> },
/// The terminal routing outcome.
Done { outcome: Py<PyRoutingOutcome> },
}
/// Async Python iterator over one Rust algorithm run.
#[pyclass(name = "_RunStream", module = "switchyard.libsy", frozen)]
struct PyRunStream {
inner: Arc<Mutex<StepStream>>,
}
impl PyRunStream {
fn new(algorithm: &Arc<dyn Algorithm>, request: Request, models: Arc<RuntimeModels>) -> Self {
// Starting a run spawns Rust work, including when Python's event loop has not started.
let stream = {
let _guard = pyo3_async_runtimes::tokio::get_runtime().enter();
Arc::clone(algorithm).run_stream(request, models, State::default())
};
Self::from_stream(stream)
}
fn from_stream(stream: StepStream) -> Self {
Self {
inner: Arc::new(Mutex::new(stream)),
}
}
}
#[pymethods]
impl PyRunStream {
fn __aiter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> {
slf
}
fn __anext__<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
let stream = Arc::clone(&self.inner);
pyo3_async_runtimes::tokio::future_into_py(py, async move {
let step = stream.lock().await.next().await;
match step {
Some(Ok(step)) => step_to_python(step),
Some(Err(error)) => Err(py_libsy_error(error)),
None => Err(PyStopAsyncIteration::new_err(())),
}
})
}
}
/// Opaque handle shared by every Rust-owned algorithm exposed to Python.
#[pyclass(name = "Algorithm", module = "switchyard.libsy", frozen)]
struct PyAlgorithm {
inner: Arc<dyn Algorithm>,
}
#[pymethods]
impl PyAlgorithm {
/// Run the algorithm as routing-time model calls followed by one terminal outcome.
///
/// Models are supplied per request, not at algorithm construction, so the
/// same algorithm object can route against a different pool on every call.
///
/// `models` maps a category name to the model ids in it, ordered best-first:
/// the algorithm picks the first entry and treats the rest as fallbacks.
/// `any`, `capable`, `efficient`, and `judge` are the categories algorithms
/// reason about. Any other name is a deployment-defined group that only a
/// custom classifier's policy can select by name. An empty category name
/// raises `ValueError`.
///
/// `subagent_models` is the same mapping for the delegated sub-agent scope.
/// Algorithms that route delegated work read it instead of `models`; when it
/// is omitted that scope is empty.
///
/// `headers`, when given, is normalized into the request's correlation
/// [`Metadata`] exactly as an HTTP host would (`Metadata::from_headers`),
/// so metadata-driven algorithms see the same signals in Python as when
/// served over HTTP.
#[pyo3(signature = (request, models, subagent_models=None, headers=None))]
fn run_stream(
&self,
request: &Bound<'_, PyAny>,
models: HashMap<String, Vec<String>>,
subagent_models: Option<HashMap<String, Vec<String>>>,
headers: Option<HashMap<String, String>>,
) -> PyResult<PyRunStream> {
let request = request_from_python(request, headers)?;
let mut runtime_models = RuntimeModels::new(category_models_from_python(models)?);
if let Some(subagent_models) = subagent_models {
runtime_models =
runtime_models.with_subagent(category_models_from_python(subagent_models)?);
}
Ok(PyRunStream::new(
&self.inner,
request,
Arc::new(runtime_models),
))
}
fn __repr__(&self) -> &'static str {
"Algorithm()"
}
}
/// Deployment algorithms parsed without constructing provider clients or reading credentials.
#[pyclass(name = "AlgorithmFactory", module = "switchyard.libsy", frozen)]
struct PyAlgorithmFactory {
inner: AlgorithmFactory,
}
#[pymethods]
impl PyAlgorithmFactory {
/// Parse a complete deployment; invalid configuration raises `ValueError`.
#[staticmethod]
fn from_toml(py: Python<'_>, source: &str) -> PyResult<Self> {
let inner = py
.detach(|| AlgorithmFactory::from_toml(source))
.map_err(|error| PyValueError::new_err(error.to_string()))?;
Ok(Self { inner })
}
/// Build independent configured handles keyed by each route's public ID.
fn build(&self, py: Python<'_>) -> PyResult<BTreeMap<String, Py<PyConfiguredAlgorithm>>> {
let algorithms = py
.detach(|| self.inner.build())
.map_err(|error| PyValueError::new_err(error.to_string()))?;
algorithms
.into_iter()
.map(|(id, (algorithm, models))| {
let handle = PyConfiguredAlgorithm {
inner: ConfiguredBackend::Bare {
algorithm,
models: Arc::new(models),
},
};
Ok((id, Py::new(py, handle)?))
})
.collect()
}
/// Build handles that check target requirements before yielding host-owned calls.
fn build_configured(
&self,
py: Python<'_>,
) -> PyResult<BTreeMap<String, Py<PyConfiguredAlgorithm>>> {
let algorithms = py
.detach(|| self.inner.build_configured())
.map_err(|error| PyValueError::new_err(error.to_string()))?;
algorithms
.into_iter()
.map(|(id, algorithm)| {
let handle = PyConfiguredAlgorithm {
inner: ConfiguredBackend::Governed(algorithm),
};
Ok((id, Py::new(py, handle)?))
})
.collect()
}
}
enum ConfiguredBackend {
Bare {
algorithm: Arc<dyn Algorithm>,
models: Arc<RuntimeModels>,
},
Governed(switchyard_runner::ConfiguredAlgorithm),
}
/// A reusable algorithm paired with its deployment's parent and subagent model groups.
/// The host applies target prompts and provider settings when serving model calls.
#[pyclass(name = "ConfiguredAlgorithm", module = "switchyard.libsy", frozen)]
struct PyConfiguredAlgorithm {
inner: ConfiguredBackend,
}
#[pymethods]
impl PyConfiguredAlgorithm {
/// All possible targets and their client/provider IDs, not the eligible pool for a request.
fn target_models<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyDict>> {
let ConfiguredBackend::Governed(algorithm) = &self.inner else {
return Err(py_libsy_error(
"target model lookup requires AlgorithmFactory.build_configured()",
));
};
let targets = PyDict::new(py);
for (name, target) in algorithm.target_models() {
let model = PyDict::new(py);
model.set_item("client", &target.client)?;
model.set_item("id", target.id.as_str())?;
targets.set_item(name.as_str(), model)?;
}
Ok(targets)
}
/// Yield host-owned model and decision calls for a normalized request.
/// Omitted labels mean no assessment; an empty collection supplies no new requirements.
/// The host authorizes `task_id`, which overrides any task ID in `headers`.
/// Libsy's per-run `State` is not carried forward, as in `Algorithm.run_stream`.
#[pyo3(signature = (request, *, headers=None, required_labels=None, task_id=None))]
fn run_stream(
&self,
py: Python<'_>,
request: &Bound<'_, PyAny>,
headers: Option<&Bound<'_, PyAny>>,
required_labels: Option<Vec<String>>,
task_id: Option<String>,
) -> PyResult<PyRunStream> {
let headers = headers.map(from_python).transpose()?;
let mut request = request_from_python(request, headers)?;
if let Some(task_id) = task_id {
request.metadata.get_or_insert_default().task_id = Some(task_id);
}
match &self.inner {
ConfiguredBackend::Bare { algorithm, models } => {
if required_labels.is_some() {
return Err(py_libsy_error(
"target requirements require AlgorithmFactory.build_configured()",
));
}
Ok(PyRunStream::new(algorithm, request, Arc::clone(models)))
}
ConfiguredBackend::Governed(algorithm) => {
if let Some(labels) = required_labels {
require_target_labels(&mut request, labels).map_err(py_libsy_error)?;
}
let _guard = pyo3_async_runtimes::tokio::get_runtime().enter();
let stream = py
.detach(|| algorithm.run_stream(request, State::default()))
.map_err(py_libsy_error)?;
Ok(PyRunStream::from_stream(stream))
}
}
}
}
fn step_to_python(step: RustStep) -> PyResult<PyStep> {
match step {
RustStep::CallDecision(call) => Python::attach(|py| {
Ok(PyStep::CallDecision {
call: Py::new(py, PyDecisionCall::new(py, *call)?)?,
})
}),
RustStep::CallModel(call) => Python::attach(|py| {
Ok(PyStep::CallModel {
call: Py::new(py, PyModelCall::new(py, *call)?)?,
})
}),
RustStep::Done(outcome) => {
let RoutingOutcome {
selected_model_ids,
request,
fallback_request,
candidate_requests,
response,
metadata,
state: _,
} = *outcome;
Python::attach(|py| {
Ok(PyStep::Done {
outcome: Py::new(
py,
PyRoutingOutcome {
metadata: metadata
.map(|inner| Py::new(py, PyOutcomeMetadata { inner }))
.transpose()?,
selected_model_ids: selected_model_ids
.iter()
.map(ToString::to_string)
.collect(),
request: to_python(py, &request.llm_request)?,
candidate_requests: to_python(
py,
&candidate_requests
.into_iter()
.map(|(id, request)| (id.to_string(), request.llm_request))
.collect::<std::collections::HashMap<_, _>>(),
)?,
fallback_request: fallback_request
.map(|request| to_python(py, &request.llm_request))
.transpose()?,
response: response
.map(|response| response_to_python(py, response.llm_response))
.transpose()?,
},
)?,
})
})
}
}
}
/// Construct the no-op reference algorithm.
#[pyfunction(name = "noop")]
fn noop_algorithm() -> PyAlgorithm {
PyAlgorithm {
inner: Arc::new(Noop {}),
}
}
/// Construct random routing over targets with optional relative weights and seed.
#[pyfunction(name = "random")]
#[pyo3(signature = (weights=None, seed=None))]
fn random_algorithm(weights: Option<Vec<f64>>, seed: Option<u64>) -> PyResult<PyAlgorithm> {
let algorithm =
Random::new(weights, seed).map_err(|other| PyValueError::new_err(other.to_string()))?;
Ok(PyAlgorithm {
inner: Arc::new(algorithm),
})
}
/// Construct LLM classifier routing from a mode config.
#[pyfunction(name = "llm_classifier")]
fn llm_classifier_algorithm(
py: Python<'_>,
config: Py<PyLlmClassifierConfig>,
) -> PyResult<PyAlgorithm> {
build_llm_classifier(config.bind(py).try_borrow()?.inner.clone())
}
/// Construct capability classifier routing.
#[pyfunction(name = "llm_task_classifier")]
#[pyo3(signature = (*, config))]
fn llm_task_classifier_algorithm(
py: Python<'_>,
config: Py<PyTaskClassifierConfig>,
) -> PyResult<PyAlgorithm> {
build_llm_classifier(LlmClassifierConfig::Capability {
config: config.bind(py).try_borrow()?.clone_core(),
})
}
fn build_llm_classifier(config: LlmClassifierConfig) -> PyResult<PyAlgorithm> {
let algorithm =
LlmTaskClassifier::new(config).map_err(|error| PyValueError::new_err(error.to_string()))?;
Ok(PyAlgorithm {
inner: Arc::new(algorithm),
})
}
/// Construct signal-driven stage routing with an optional LLM classifier fallback.
#[pyfunction(name = "stage_router")]
#[pyo3(signature = (
*,
picker,
confidence_threshold,
recent_window=None,
escalation_note=None,
deescalation_note=None,
only_on_wrong_signal_escalation=true,
capable_system_prompt=None,
efficient_system_prompt=None,
tool_semantics=None,
classifier=None
))]
#[allow(clippy::too_many_arguments)]
fn stage_router_algorithm(
py: Python<'_>,
picker: &str,
confidence_threshold: f64,
recent_window: Option<usize>,
escalation_note: Option<String>,
deescalation_note: Option<String>,
only_on_wrong_signal_escalation: bool,
capable_system_prompt: Option<String>,
efficient_system_prompt: Option<String>,
tool_semantics: Option<HashMap<String, Vec<String>>>,
classifier: Option<Py<PyLlmFallback>>,
) -> PyResult<PyAlgorithm> {
let mode = match picker {
"capable_first" => PickerMode::CapableFirst,
"efficient_first" => PickerMode::EfficientFirst,
other => {
return Err(PyValueError::new_err(format!(
"picker must be 'capable_first' or 'efficient_first', got {other:?}"
)));
}
};
let mut config = StageRouterConfig::new(mode, confidence_threshold);
config.recent_window = recent_window;
config.handoff_notes = match (escalation_note, deescalation_note) {
(Some(escalation), deescalation) => Some(HandoffNoteConfig::new(
escalation,
deescalation,
only_on_wrong_signal_escalation,
)),
(None, Some(_)) => {
return Err(PyValueError::new_err(
"deescalation_note requires escalation_note",
));
}
(None, None) => None,
};
config.capable_system_prompt = capable_system_prompt;
config.efficient_system_prompt = efficient_system_prompt;
if let Some(mut semantics) = tool_semantics {
config.tool_semantics = ToolSemantics {
observe: semantics.remove("observe").unwrap_or_default(),
mutate: semantics.remove("mutate").unwrap_or_default(),
plan: semantics.remove("plan").unwrap_or_default(),
new: semantics.remove("new").unwrap_or_default(),
};
if let Some(category) = semantics.keys().next() {
return Err(PyValueError::new_err(format!(
"unknown tool_semantics category {category:?}; expected observe, mutate, plan, or new"
)));
}
}
config.llm_fallback = classifier
.map(|classifier| classifier.bind(py).try_borrow()?.clone_core(py))
.transpose()?;
let algorithm =
StageRouter::new(config).map_err(|error| PyValueError::new_err(error.to_string()))?;
Ok(PyAlgorithm {
inner: Arc::new(algorithm),
})
}
pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
let libsy_module = PyModule::new(module.py(), "libsy")?;
libsy_module.add_class::<PyAlgorithm>()?;
libsy_module.add_class::<PyAlgorithmFactory>()?;
libsy_module.add_class::<PyConfiguredAlgorithm>()?;
libsy_module.add_class::<PyCustomClassifierConfig>()?;
libsy_module.add_class::<PyDecisionCall>()?;
libsy_module.add_class::<PyDeescalationConfig>()?;
libsy_module.add_class::<PyEscalationClassifierConfig>()?;
libsy_module.add_class::<PyLlmClassifierConfig>()?;
libsy_module.add_class::<PyLlmFallback>()?;
libsy_module.add_class::<PyLlmResponse>()?;
libsy_module.add_class::<PyLlmResponseStream>()?;
libsy_module.add_class::<PyModelCall>()?;
libsy_module.add_class::<PyOutcomeMetadata>()?;
libsy_module.add_class::<PyRunStream>()?;
libsy_module.add_class::<PyRoutingOutcome>()?;
libsy_module.add_class::<PyStep>()?;
libsy_module.add_class::<PyTaskClassifierConfig>()?;
libsy_module.add_function(wrap_pyfunction!(noop_algorithm, &libsy_module)?)?;
libsy_module.add_function(wrap_pyfunction!(random_algorithm, &libsy_module)?)?;
libsy_module.add_function(wrap_pyfunction!(llm_classifier_algorithm, &libsy_module)?)?;
libsy_module.add_function(wrap_pyfunction!(
llm_task_classifier_algorithm,
&libsy_module
)?)?;
libsy_module.add_function(wrap_pyfunction!(stage_router_algorithm, &libsy_module)?)?;
libsy_module.add(
"ContextWindowExceededError",
module.getattr("ContextWindowExceededError")?,
)?;
libsy_module.add("LibsyError", module.getattr("LibsyError")?)?;
module.add_submodule(&libsy_module)?;
Ok(())
}