Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(python): support configured target requirements (#1249)
Alex Fournier9895f2cb1574
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rsindex ee2e7bdbf..b9a6b4106 100644--- a/crates/switchyard-py/src/libsy_bindings.rs+++ b/crates/switchyard-py/src/libsy_bindings.rs@@ -11,6 +11,7 @@ use http::HeaderMap;use http::header::{HeaderName, HeaderValue};use pyo3::exceptions::{PyBaseException, PyStopAsyncIteration, PyTypeError, PyValueError};use pyo3::prelude::*;+use pyo3::types::PyDict;use serde_json::Value;use switchyard_libsy::{Algorithm, CallDecision, CallModel, CapabilityJudgeConfig, ClassifierContractConfig,@@ -25,7 +26,7 @@ use switchyard_protocol::{Category, DecisionResponse, LlmClientError, LlmResponse, LlmResponseStream,LlmResponseStreamEvent, Metadata, ModelId, Request, Response,};-use switchyard_runner::AlgorithmFactory;+use switchyard_runner::{AlgorithmFactory, require_target_labels};use tokio::sync::Mutex;use crate::errors::{ContextWindowExceededError, py_libsy_error};@@ -892,6 +893,10 @@ impl PyRunStream {let _guard = pyo3_async_runtimes::tokio::get_runtime().enter();Arc::clone(algorithm).run_stream(request, models, State::default())};+ Self::from_stream(stream)+ }++ fn from_stream(stream: StepStream) -> Self {Self {inner: Arc::new(Mutex::new(stream)),}@@ -996,45 +1001,109 @@ impl PyAlgorithmFactory {algorithms.into_iter().map(|(id, (algorithm, models))| {- Ok((- id,- Py::new(- py,- PyConfiguredAlgorithm {- algorithm,- models: Arc::new(models),- },- )?,- ))+ let handle = PyConfiguredAlgorithm {+ inner: ConfiguredBackend::Bare {+ algorithm,+ models: Arc::new(models),+ },+ };+ Ok((id, Py::new(py, handle)?))+ })+ .collect()+ }++ /// Build handles that check target requirements before yielding host-owned calls.+ fn build_configured(+ &self,+ py: Python<'_>,+ ) -> PyResult<BTreeMap<String, Py<PyConfiguredAlgorithm>>> {+ let algorithms = py+ .detach(|| self.inner.build_configured())+ .map_err(|error| PyValueError::new_err(error.to_string()))?;+ algorithms+ .into_iter()+ .map(|(id, algorithm)| {+ let handle = PyConfiguredAlgorithm {+ inner: ConfiguredBackend::Governed(algorithm),+ };+ Ok((id, Py::new(py, handle)?))}).collect()}}+enum ConfiguredBackend {+ Bare {+ algorithm: Arc<dyn Algorithm>,+ models: Arc<RuntimeModels>,+ },+ Governed(switchyard_runner::ConfiguredAlgorithm),+}+/// A reusable algorithm paired with its deployment's parent and subagent model groups./// The host applies target prompts and provider settings when serving model calls.#[pyclass(name = "ConfiguredAlgorithm", module = "switchyard.libsy", frozen)]struct PyConfiguredAlgorithm {- algorithm: Arc<dyn Algorithm>,- models: Arc<RuntimeModels>,+ inner: ConfiguredBackend,}#[pymethods]impl PyConfiguredAlgorithm {+ /// All possible targets and their client/provider IDs, not the eligible pool for a request.+ fn target_models<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyDict>> {+ let ConfiguredBackend::Governed(algorithm) = &self.inner else {+ return Err(py_libsy_error(+ "target model lookup requires AlgorithmFactory.build_configured()",+ ));+ };+ let targets = PyDict::new(py);+ for (name, target) in algorithm.target_models() {+ let model = PyDict::new(py);+ model.set_item("client", &target.client)?;+ model.set_item("id", target.id.as_str())?;+ targets.set_item(name.as_str(), model)?;+ }+ Ok(targets)+ }+/// Yield host-owned model and decision calls for a normalized request.+ /// Omitted labels mean no assessment; an empty collection supplies no new requirements.+ /// The host authorizes `task_id`, which overrides any task ID in `headers`./// Libsy's per-run `State` is not carried forward, as in `Algorithm.run_stream`.- #[pyo3(signature = (request, *, headers=None))]+ #[pyo3(signature = (request, *, headers=None, required_labels=None, task_id=None))]fn run_stream(&self,+ py: Python<'_>,request: &Bound<'_, PyAny>,headers: Option<&Bound<'_, PyAny>>,+ required_labels: Option<Vec<String>>,+ task_id: Option<String>,) -> PyResult<PyRunStream> {let headers = headers.map(from_python).transpose()?;- Ok(PyRunStream::new(- &self.algorithm,- request_from_python(request, headers)?,- Arc::clone(&self.models),- ))+ let mut request = request_from_python(request, headers)?;+ if let Some(task_id) = task_id {+ request.metadata.get_or_insert_default().task_id = Some(task_id);+ }+ match &self.inner {+ ConfiguredBackend::Bare { algorithm, models } => {+ if required_labels.is_some() {+ return Err(py_libsy_error(+ "target requirements require AlgorithmFactory.build_configured()",+ ));+ }+ Ok(PyRunStream::new(algorithm, request, Arc::clone(models)))+ }+ ConfiguredBackend::Governed(algorithm) => {+ if let Some(labels) = required_labels {+ require_target_labels(&mut request, labels).map_err(py_libsy_error)?;+ }+ let _guard = pyo3_async_runtimes::tokio::get_runtime().enter();+ let stream = py+ .detach(|| algorithm.run_stream(request, State::default()))+ .map_err(py_libsy_error)?;+ Ok(PyRunStream::from_stream(stream))+ }+ }}}diff --git a/docs/reference/host_integration.md b/docs/reference/host_integration.mdindex 393a9deb0..7f60d1682 100644--- a/docs/reference/host_integration.md+++ b/docs/reference/host_integration.md@@ -123,6 +123,55 @@ asyncio.run(main())`uv run route.py` prints `Selected: Efficient` without making a network call.+### Target requirements (development)++With matching development bindings, load deployment TOML that declares target+labels and route eligibility. Build once and reuse the returned handles:++```python+from switchyard.libsy import AlgorithmFactory++routes = AlgorithmFactory.from_toml(toml).build_configured()+coding = routes["switchyard/coding"]+targets = coding.target_models()++steps = coding.run_stream(+ request,+ required_labels=["private", "customer_data"],+ task_id=trusted_task_id,+)+```++For each `Step.CallModel`, resolve the returned target before calling your client.+Here `clients` is supplied by the gateway and keyed by the TOML client aliases,+not a new Switchyard interface. Its clients follow the+[Python host example](../../examples/libsy.py): they accept a normalized request+and provider model ID, and return `LlmResponse.Agg` or `LlmResponse.Stream`.++```python+async def serve_model(call):+ target = targets[call.models[0]]+ client = clients[target["client"]]+ await call.respond(client.call(call.request, target["id"]))+```++Use the first routing-time candidate and pass client failures back to Switchyard.+Typed Decision calls use `targets[call.model]` and their own Decision handler.+For a final answer candidate, use `targets[outcome.selected_model_ids[0]]`.+The lookup contains all possible targets, not the eligible pool for this request;+use only keys returned by calls and outcomes. `build()` handles reject this lookup.++The host supplies the assessment and an authorized task ID. An explicit task ID+overrides a task ID from headers. Omitted labels mean no assessment; `[]` supplies+no new requirements. In task scope, earlier requirements remain in effect.+Invalid requirements or missing approved targets fail before any routing call.++Consume `steps` through the same model and Decision call handlers. Switchyard+does not create provider clients or read their credentials. The host owns target+prompts, provider settings and transport, and must keep answer attempts and+fallbacks within the returned candidates. Existing `build()` handles do not+accept target requirements. Per-run libsy state is not carried between Python calls.+### Concurrent calls and cancellationThe client coroutine returns `LlmResponse.Agg` or `LlmResponse.Stream`.diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.pyindex 9fa46f3aa..267506cc6 100644--- a/switchyard_rust/libsy.py+++ b/switchyard_rust/libsy.py@@ -306,6 +306,10 @@ if TYPE_CHECKING:"""Build independent handles keyed by the public route IDs."""...+ def build_configured(self) -> dict[str, ConfiguredAlgorithm]:+ """Build handles that check target requirements before yielding host calls."""+ ...+@finalclass ConfiguredAlgorithm:"""LLM calls and outcomes use target names; the host resolves provider IDs and settings.@@ -313,13 +317,30 @@ if TYPE_CHECKING:Decision calls use provider IDs."""+ def target_models(self) -> dict[str, dict[str, str]]:+ """Map every possible target key to its ``client`` and provider ``id``.++ Requires ``build_configured()``. This includes ineligible targets;+ execute only candidates returned by a call or final outcome.+ """+ ...+def run_stream(self,request: Mapping[str, object],*,headers: Mapping[str, str] | None = None,+ required_labels: Sequence[str] | None = None,+ task_id: str | None = None,) -> AsyncIterator[Step.CallModel | Step.CallDecision | Step.Done]:- """Yield host calls and an outcome without carrying libsy's per-run State forward."""+ """Yield host calls; target requirements need a ``build_configured()`` handle.++ Omitted labels mean no assessment; ``[]`` supplies no new requirements.+ Reuse the handle for retained task requirements. The host authorizes+ ``task_id``, which overrides a task ID from headers. Libsy's per-run+ State is not carried forward. Answers and fallbacks must use the+ returned candidates.+ """...def noop() -> Algorithm: ...diff --git a/tests/test_algorithm_factory.py b/tests/test_algorithm_factory.pyindex 48d2e95d0..c54e0de31 100644--- a/tests/test_algorithm_factory.py+++ b/tests/test_algorithm_factory.py@@ -1,36 +1,203 @@# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.# SPDX-License-Identifier: Apache-2.0-"""Smoke test for the Python algorithm factory."""+"""Client-free configured algorithms and trusted target requirements."""-from switchyard.libsy import AlgorithmFactory, ConfiguredAlgorithm, Step+from __future__ import annotations++import json+from collections.abc import AsyncIterator+from typing import Literal++import pytest++from switchyard.libsy import (+ AlgorithmFactory,+ ConfiguredAlgorithm,+ LibsyError,+ LlmResponse,+ RoutingOutcome,+ Step,+)async def test_factory_builds_a_usable_python_handle() -> None:"""Configured handles run without a separate model map."""- routes = AlgorithmFactory.from_toml("""+ source = deployment().split("[routes.coding.eligibility]")[0]+ route = AlgorithmFactory.from_toml(source).build()["switchyard/coding"]+ assert isinstance(route, ConfiguredAlgorithm)+ with pytest.raises(LibsyError, match="build_configured"):+ route.target_models()+ with pytest.raises(LibsyError, match="build_configured"):+ route.run_stream(request_body(), required_labels=[])+ outcome = await finish(route.run_stream(request_body()))+ assert outcome.selected_model_ids == ["fast", "capable"]+++def deployment(*, scope: str = "request", judge: Literal["llm", "decision"] | None = None) -> str:+ targets = "\n".join(+ f'{name} = {{ id = "provider/{name}", llm_client = "{client}", labels = {json.dumps(labels)} }}'+ for name, client, labels in [+ ("fast", "standard", []),+ ("capable", "standard", []),+ ("private_fast", "approved", ["private"]),+ ("private_capable", "approved", ["private"]),+ ("both_fast", "approved", ["private", "customer_data"]),+ ("both_capable", "approved", ["private", "customer_data"]),+ ("judge", "standard", []),+ ("approved_judge", "approved", ["private", "customer_data"]),+ ]+ )+ algorithm = (+ 'type = "stage_router"\nefficient_target = "fast"\ncapable_target = "capable"\n'+ 'picker = "efficient_first"\nconfidence_threshold = 0.6'+ )+ alternative = ""+ if judge is not None:+ is_decision = judge == "decision"+ judge_target = "judge_decision" if is_decision else "judge"+ settings = "[routes.coding.decision]\ncutoff = 0.7" if is_decision else "base_threshold = 0.5"+ algorithm = (+ f'type = "llm_classifier"\nclassifier_target = "{judge_target}"\n'+ f'strong_target = "capable"\nweak_target = "fast"\nfail_open = false\n{settings}'+ )+ replacement = "approved_decision" if is_decision else "approved_judge"+ alternative = f'{judge_target} = ["{replacement}"]'+ return f"""schema_version = 1-[llm_clients.host]+[llm_clients.standard]+format = "openai_chat"+base_url = "https://standard.invalid/v1"+api_key_env = "SWITCHYARD_PYTHON_FACTORY_UNUSED_KEY"+[llm_clients.approved]format = "openai_chat"-base_url = "https://host.invalid/v1"+base_url = "https://approved.invalid/v1"+api_key_env = "SWITCHYARD_PYTHON_FACTORY_UNUSED_KEY"[targets]-fast = { id = "gateway/fast", llm_client = "host" }-[routes.main]-id = "switchyard/main"-type = "passthrough"-target = "fast"-""").build()- route = routes["switchyard/main"]- assert isinstance(route, ConfiguredAlgorithm)- steps = [- step- async for step in route.run_stream(- {- "model": "switchyard/main",- "messages": [{"role": "user", "content": [{"type": "text", "text": "hello"}]}],- }+{targets}+[decision_clients.scoring]+format = "openai_decisions"+endpoint = "https://host.invalid/v1/decisions"+api_key_env = "SWITCHYARD_PYTHON_FACTORY_UNUSED_KEY"+timeout_ms = 1000+[decision_targets.judge_decision]+id = "decision/standard"+decision_client = "scoring"+[decision_targets.approved_decision]+id = "decision/approved"+decision_client = "scoring"+labels = ["private", "customer_data"]+[routes.coding]+id = "switchyard/coding"+{algorithm}+[routes.coding.eligibility]+require_explicit_requirements = true+scope = "{scope}"+[routes.coding.eligibility.target_alternatives]+fast = ["private_fast", "both_fast"]+capable = ["private_capable", "both_capable"]+{alternative}+"""+++def request_body() -> dict[str, object]:+ return {"messages": [{"role": "user", "content": [{"type": "text", "text": "hello"}]}]}+++async def finish(+ stream: AsyncIterator[Step.CallModel | Step.CallDecision | Step.Done],+) -> RoutingOutcome:+ step = await anext(stream)+ assert isinstance(step, Step.Done)+ with pytest.raises(StopAsyncIteration):+ await anext(stream)+ return step.outcome+++def test_requirements_are_rejected_before_returning_routing_work() -> None:+ factory = AlgorithmFactory.from_toml(deployment())+ with pytest.raises(ValueError, match="Runner admission"):+ factory.build()+ route = factory.build_configured()["switchyard/coding"]+ with pytest.raises(LibsyError, match="explicit target requirements"):+ route.run_stream(request_body())+ with pytest.raises(LibsyError, match="required target"):+ route.run_stream(request_body(), required_labels=["unapproved"])+++async def test_alternatives_and_task_requirements_use_the_trusted_task_id() -> None:+ route = AlgorithmFactory.from_toml(deployment(scope="task")).build_configured()[+ "switchyard/coding"+ ]+ targets = route.target_models()+ with pytest.raises(LibsyError, match="valid task id"):+ route.run_stream(request_body(), required_labels=[])+ for task, labels, expected in [+ ("shared", ["private"], "private_fast"),+ ("shared", ["customer_data"], "both_fast"),+ ("shared", [], "both_fast"),+ ("other", [], "fast"),+ ]:+ outcome = await finish(+ route.run_stream(+ request_body(),+ required_labels=labels,+ task_id=task,+ headers={"x-switchyard-task-id": "caller-controlled"},+ ))+ assert outcome.selected_model_ids[0] == expected+ assert targets[outcome.selected_model_ids[0]] == {+ "client": "standard" if expected == "fast" else "approved",+ "id": f"provider/{expected}",+ }++[email protected]("judge", ["llm", "decision"])+async def test_approved_judge_calls_are_executed_by_the_python_host(+ judge: Literal["llm", "decision"],+) -> None:+ route = AlgorithmFactory.from_toml(deployment(judge=judge)).build_configured()[+ "switchyard/coding"]- assert len(steps) == 1- assert isinstance(steps[0], Step.Done)- assert steps[0].outcome.selected_model_ids == ["fast"]+ targets = route.target_models()+ stream = route.run_stream(request_body(), required_labels=["private", "customer_data"])+ step = await anext(stream)+ if judge == "decision":+ assert isinstance(step, Step.CallDecision)+ assert step.call.model == step.call.request["model"] == "decision/approved"+ assert targets[step.call.model] == {"client": "scoring", "id": "decision/approved"}+ choice = {+ "type": "choice",+ "data": {+ "selected": "efficient",+ "probabilities": {"efficient": 0.9, "capable": 0.1},+ },+ }+ response = {"answers": {"route": {"value": choice}}}+ else:+ assert isinstance(step, Step.CallModel)+ assert step.call.models == ["approved_judge"]+ assert step.call.request["model"] == "approved_judge"+ target = targets[step.call.models[0]]+ assert target == {"client": "approved", "id": "provider/approved_judge"}+ verdict = {+ "crux": "bounded task",+ "primary_rule": "SUP-1",+ "capability_boundary": "supported",+ "p_solve": 0.9,+ }+ message = {+ "role": "assistant",+ "content": [{"type": "text", "text": json.dumps(verdict)}],+ "stop_reason": "end_turn",+ }+ response = LlmResponse.Agg({"model": target["id"], "outputs": [message]})++ async def reply() -> object:+ return response++ await step.call.respond(reply())+ outcome = await finish(stream)+ assert outcome.selected_model_ids == ["both_fast", "both_capable"]+ assert all(targets[key]["client"] == "approved" for key in outcome.selected_model_ids)