Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(python): support configured target requirements (#1249)

Alex Fournier9895f2cb1574
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index ee2e7bdbf..b9a6b4106 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -11,6 +11,7 @@ use http::HeaderMap;
use http::header::{HeaderName, HeaderValue};
use pyo3::exceptions::{PyBaseException, PyStopAsyncIteration, PyTypeError, PyValueError};
use pyo3::prelude::*;
+use pyo3::types::PyDict;
use serde_json::Value;
use switchyard_libsy::{
Algorithm, CallDecision, CallModel, CapabilityJudgeConfig, ClassifierContractConfig,
@@ -25,7 +26,7 @@ use switchyard_protocol::{
Category, DecisionResponse, LlmClientError, LlmResponse, LlmResponseStream,
LlmResponseStreamEvent, Metadata, ModelId, Request, Response,
};
-use switchyard_runner::AlgorithmFactory;
+use switchyard_runner::{AlgorithmFactory, require_target_labels};
use tokio::sync::Mutex;
use crate::errors::{ContextWindowExceededError, py_libsy_error};
@@ -892,6 +893,10 @@ impl PyRunStream {
let _guard = pyo3_async_runtimes::tokio::get_runtime().enter();
Arc::clone(algorithm).run_stream(request, models, State::default())
};
+ Self::from_stream(stream)
+ }
+
+ fn from_stream(stream: StepStream) -> Self {
Self {
inner: Arc::new(Mutex::new(stream)),
}
@@ -996,45 +1001,109 @@ impl PyAlgorithmFactory {
algorithms
.into_iter()
.map(|(id, (algorithm, models))| {
- Ok((
- id,
- Py::new(
- py,
- PyConfiguredAlgorithm {
- algorithm,
- models: Arc::new(models),
- },
- )?,
- ))
+ let handle = PyConfiguredAlgorithm {
+ inner: ConfiguredBackend::Bare {
+ algorithm,
+ models: Arc::new(models),
+ },
+ };
+ Ok((id, Py::new(py, handle)?))
+ })
+ .collect()
+ }
+
+ /// Build handles that check target requirements before yielding host-owned calls.
+ fn build_configured(
+ &self,
+ py: Python<'_>,
+ ) -> PyResult<BTreeMap<String, Py<PyConfiguredAlgorithm>>> {
+ let algorithms = py
+ .detach(|| self.inner.build_configured())
+ .map_err(|error| PyValueError::new_err(error.to_string()))?;
+ algorithms
+ .into_iter()
+ .map(|(id, algorithm)| {
+ let handle = PyConfiguredAlgorithm {
+ inner: ConfiguredBackend::Governed(algorithm),
+ };
+ Ok((id, Py::new(py, handle)?))
})
.collect()
}
}
+enum ConfiguredBackend {
+ Bare {
+ algorithm: Arc<dyn Algorithm>,
+ models: Arc<RuntimeModels>,
+ },
+ Governed(switchyard_runner::ConfiguredAlgorithm),
+}
+
/// A reusable algorithm paired with its deployment's parent and subagent model groups.
/// The host applies target prompts and provider settings when serving model calls.
#[pyclass(name = "ConfiguredAlgorithm", module = "switchyard.libsy", frozen)]
struct PyConfiguredAlgorithm {
- algorithm: Arc<dyn Algorithm>,
- models: Arc<RuntimeModels>,
+ inner: ConfiguredBackend,
}
#[pymethods]
impl PyConfiguredAlgorithm {
+ /// All possible targets and their client/provider IDs, not the eligible pool for a request.
+ fn target_models<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyDict>> {
+ let ConfiguredBackend::Governed(algorithm) = &self.inner else {
+ return Err(py_libsy_error(
+ "target model lookup requires AlgorithmFactory.build_configured()",
+ ));
+ };
+ let targets = PyDict::new(py);
+ for (name, target) in algorithm.target_models() {
+ let model = PyDict::new(py);
+ model.set_item("client", &target.client)?;
+ model.set_item("id", target.id.as_str())?;
+ targets.set_item(name.as_str(), model)?;
+ }
+ Ok(targets)
+ }
+
/// Yield host-owned model and decision calls for a normalized request.
+ /// Omitted labels mean no assessment; an empty collection supplies no new requirements.
+ /// The host authorizes `task_id`, which overrides any task ID in `headers`.
/// Libsy's per-run `State` is not carried forward, as in `Algorithm.run_stream`.
- #[pyo3(signature = (request, *, headers=None))]
+ #[pyo3(signature = (request, *, headers=None, required_labels=None, task_id=None))]
fn run_stream(
&self,
+ py: Python<'_>,
request: &Bound<'_, PyAny>,
headers: Option<&Bound<'_, PyAny>>,
+ required_labels: Option<Vec<String>>,
+ task_id: Option<String>,
) -> PyResult<PyRunStream> {
let headers = headers.map(from_python).transpose()?;
- Ok(PyRunStream::new(
- &self.algorithm,
- request_from_python(request, headers)?,
- Arc::clone(&self.models),
- ))
+ let mut request = request_from_python(request, headers)?;
+ if let Some(task_id) = task_id {
+ request.metadata.get_or_insert_default().task_id = Some(task_id);
+ }
+ match &self.inner {
+ ConfiguredBackend::Bare { algorithm, models } => {
+ if required_labels.is_some() {
+ return Err(py_libsy_error(
+ "target requirements require AlgorithmFactory.build_configured()",
+ ));
+ }
+ Ok(PyRunStream::new(algorithm, request, Arc::clone(models)))
+ }
+ ConfiguredBackend::Governed(algorithm) => {
+ if let Some(labels) = required_labels {
+ require_target_labels(&mut request, labels).map_err(py_libsy_error)?;
+ }
+ let _guard = pyo3_async_runtimes::tokio::get_runtime().enter();
+ let stream = py
+ .detach(|| algorithm.run_stream(request, State::default()))
+ .map_err(py_libsy_error)?;
+ Ok(PyRunStream::from_stream(stream))
+ }
+ }
}
}
diff --git a/docs/reference/host_integration.md b/docs/reference/host_integration.md
index 393a9deb0..7f60d1682 100644
--- a/docs/reference/host_integration.md
+++ b/docs/reference/host_integration.md
@@ -123,6 +123,55 @@ asyncio.run(main())
`uv run route.py` prints `Selected: Efficient` without making a network call.
+### Target requirements (development)
+
+With matching development bindings, load deployment TOML that declares target
+labels and route eligibility. Build once and reuse the returned handles:
+
+```python
+from switchyard.libsy import AlgorithmFactory
+
+routes = AlgorithmFactory.from_toml(toml).build_configured()
+coding = routes["switchyard/coding"]
+targets = coding.target_models()
+
+steps = coding.run_stream(
+ request,
+ required_labels=["private", "customer_data"],
+ task_id=trusted_task_id,
+)
+```
+
+For each `Step.CallModel`, resolve the returned target before calling your client.
+Here `clients` is supplied by the gateway and keyed by the TOML client aliases,
+not a new Switchyard interface. Its clients follow the
+[Python host example](../../examples/libsy.py): they accept a normalized request
+and provider model ID, and return `LlmResponse.Agg` or `LlmResponse.Stream`.
+
+```python
+async def serve_model(call):
+ target = targets[call.models[0]]
+ client = clients[target["client"]]
+ await call.respond(client.call(call.request, target["id"]))
+```
+
+Use the first routing-time candidate and pass client failures back to Switchyard.
+Typed Decision calls use `targets[call.model]` and their own Decision handler.
+For a final answer candidate, use `targets[outcome.selected_model_ids[0]]`.
+The lookup contains all possible targets, not the eligible pool for this request;
+use only keys returned by calls and outcomes. `build()` handles reject this lookup.
+
+The host supplies the assessment and an authorized task ID. An explicit task ID
+overrides a task ID from headers. Omitted labels mean no assessment; `[]` supplies
+no new requirements. In task scope, earlier requirements remain in effect.
+Invalid requirements or missing approved targets fail before any routing call.
+
+Consume `steps` through the same model and Decision call handlers. Switchyard
+does not create provider clients or read their credentials. The host owns target
+prompts, provider settings and transport, and must keep answer attempts and
+fallbacks within the returned candidates. Existing `build()` handles do not
+accept target requirements. Per-run libsy state is not carried between Python calls.
+
### Concurrent calls and cancellation
The client coroutine returns `LlmResponse.Agg` or `LlmResponse.Stream`.
diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py
index 9fa46f3aa..267506cc6 100644
--- a/switchyard_rust/libsy.py
+++ b/switchyard_rust/libsy.py
@@ -306,6 +306,10 @@ if TYPE_CHECKING:
"""Build independent handles keyed by the public route IDs."""
...
+ def build_configured(self) -> dict[str, ConfiguredAlgorithm]:
+ """Build handles that check target requirements before yielding host calls."""
+ ...
+
@final
class ConfiguredAlgorithm:
"""LLM calls and outcomes use target names; the host resolves provider IDs and settings.
@@ -313,13 +317,30 @@ if TYPE_CHECKING:
Decision calls use provider IDs.
"""
+ def target_models(self) -> dict[str, dict[str, str]]:
+ """Map every possible target key to its ``client`` and provider ``id``.
+
+ Requires ``build_configured()``. This includes ineligible targets;
+ execute only candidates returned by a call or final outcome.
+ """
+ ...
+
def run_stream(
self,
request: Mapping[str, object],
*,
headers: Mapping[str, str] | None = None,
+ required_labels: Sequence[str] | None = None,
+ task_id: str | None = None,
) -> AsyncIterator[Step.CallModel | Step.CallDecision | Step.Done]:
- """Yield host calls and an outcome without carrying libsy's per-run State forward."""
+ """Yield host calls; target requirements need a ``build_configured()`` handle.
+
+ Omitted labels mean no assessment; ``[]`` supplies no new requirements.
+ Reuse the handle for retained task requirements. The host authorizes
+ ``task_id``, which overrides a task ID from headers. Libsy's per-run
+ State is not carried forward. Answers and fallbacks must use the
+ returned candidates.
+ """
...
def noop() -> Algorithm: ...
diff --git a/tests/test_algorithm_factory.py b/tests/test_algorithm_factory.py
index 48d2e95d0..c54e0de31 100644
--- a/tests/test_algorithm_factory.py
+++ b/tests/test_algorithm_factory.py
@@ -1,36 +1,203 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-"""Smoke test for the Python algorithm factory."""
+"""Client-free configured algorithms and trusted target requirements."""
-from switchyard.libsy import AlgorithmFactory, ConfiguredAlgorithm, Step
+from __future__ import annotations
+
+import json
+from collections.abc import AsyncIterator
+from typing import Literal
+
+import pytest
+
+from switchyard.libsy import (
+ AlgorithmFactory,
+ ConfiguredAlgorithm,
+ LibsyError,
+ LlmResponse,
+ RoutingOutcome,
+ Step,
+)
async def test_factory_builds_a_usable_python_handle() -> None:
"""Configured handles run without a separate model map."""
- routes = AlgorithmFactory.from_toml("""
+ source = deployment().split("[routes.coding.eligibility]")[0]
+ route = AlgorithmFactory.from_toml(source).build()["switchyard/coding"]
+ assert isinstance(route, ConfiguredAlgorithm)
+ with pytest.raises(LibsyError, match="build_configured"):
+ route.target_models()
+ with pytest.raises(LibsyError, match="build_configured"):
+ route.run_stream(request_body(), required_labels=[])
+ outcome = await finish(route.run_stream(request_body()))
+ assert outcome.selected_model_ids == ["fast", "capable"]
+
+
+def deployment(*, scope: str = "request", judge: Literal["llm", "decision"] | None = None) -> str:
+ targets = "\n".join(
+ f'{name} = {{ id = "provider/{name}", llm_client = "{client}", labels = {json.dumps(labels)} }}'
+ for name, client, labels in [
+ ("fast", "standard", []),
+ ("capable", "standard", []),
+ ("private_fast", "approved", ["private"]),
+ ("private_capable", "approved", ["private"]),
+ ("both_fast", "approved", ["private", "customer_data"]),
+ ("both_capable", "approved", ["private", "customer_data"]),
+ ("judge", "standard", []),
+ ("approved_judge", "approved", ["private", "customer_data"]),
+ ]
+ )
+ algorithm = (
+ 'type = "stage_router"\nefficient_target = "fast"\ncapable_target = "capable"\n'
+ 'picker = "efficient_first"\nconfidence_threshold = 0.6'
+ )
+ alternative = ""
+ if judge is not None:
+ is_decision = judge == "decision"
+ judge_target = "judge_decision" if is_decision else "judge"
+ settings = "[routes.coding.decision]\ncutoff = 0.7" if is_decision else "base_threshold = 0.5"
+ algorithm = (
+ f'type = "llm_classifier"\nclassifier_target = "{judge_target}"\n'
+ f'strong_target = "capable"\nweak_target = "fast"\nfail_open = false\n{settings}'
+ )
+ replacement = "approved_decision" if is_decision else "approved_judge"
+ alternative = f'{judge_target} = ["{replacement}"]'
+ return f"""
schema_version = 1
-[llm_clients.host]
+[llm_clients.standard]
+format = "openai_chat"
+base_url = "https://standard.invalid/v1"
+api_key_env = "SWITCHYARD_PYTHON_FACTORY_UNUSED_KEY"
+[llm_clients.approved]
format = "openai_chat"
-base_url = "https://host.invalid/v1"
+base_url = "https://approved.invalid/v1"
+api_key_env = "SWITCHYARD_PYTHON_FACTORY_UNUSED_KEY"
[targets]
-fast = { id = "gateway/fast", llm_client = "host" }
-[routes.main]
-id = "switchyard/main"
-type = "passthrough"
-target = "fast"
-""").build()
- route = routes["switchyard/main"]
- assert isinstance(route, ConfiguredAlgorithm)
- steps = [
- step
- async for step in route.run_stream(
- {
- "model": "switchyard/main",
- "messages": [{"role": "user", "content": [{"type": "text", "text": "hello"}]}],
- }
+{targets}
+[decision_clients.scoring]
+format = "openai_decisions"
+endpoint = "https://host.invalid/v1/decisions"
+api_key_env = "SWITCHYARD_PYTHON_FACTORY_UNUSED_KEY"
+timeout_ms = 1000
+[decision_targets.judge_decision]
+id = "decision/standard"
+decision_client = "scoring"
+[decision_targets.approved_decision]
+id = "decision/approved"
+decision_client = "scoring"
+labels = ["private", "customer_data"]
+[routes.coding]
+id = "switchyard/coding"
+{algorithm}
+[routes.coding.eligibility]
+require_explicit_requirements = true
+scope = "{scope}"
+[routes.coding.eligibility.target_alternatives]
+fast = ["private_fast", "both_fast"]
+capable = ["private_capable", "both_capable"]
+{alternative}
+"""
+
+
+def request_body() -> dict[str, object]:
+ return {"messages": [{"role": "user", "content": [{"type": "text", "text": "hello"}]}]}
+
+
+async def finish(
+ stream: AsyncIterator[Step.CallModel | Step.CallDecision | Step.Done],
+) -> RoutingOutcome:
+ step = await anext(stream)
+ assert isinstance(step, Step.Done)
+ with pytest.raises(StopAsyncIteration):
+ await anext(stream)
+ return step.outcome
+
+
+def test_requirements_are_rejected_before_returning_routing_work() -> None:
+ factory = AlgorithmFactory.from_toml(deployment())
+ with pytest.raises(ValueError, match="Runner admission"):
+ factory.build()
+ route = factory.build_configured()["switchyard/coding"]
+ with pytest.raises(LibsyError, match="explicit target requirements"):
+ route.run_stream(request_body())
+ with pytest.raises(LibsyError, match="required target"):
+ route.run_stream(request_body(), required_labels=["unapproved"])
+
+
+async def test_alternatives_and_task_requirements_use_the_trusted_task_id() -> None:
+ route = AlgorithmFactory.from_toml(deployment(scope="task")).build_configured()[
+ "switchyard/coding"
+ ]
+ targets = route.target_models()
+ with pytest.raises(LibsyError, match="valid task id"):
+ route.run_stream(request_body(), required_labels=[])
+ for task, labels, expected in [
+ ("shared", ["private"], "private_fast"),
+ ("shared", ["customer_data"], "both_fast"),
+ ("shared", [], "both_fast"),
+ ("other", [], "fast"),
+ ]:
+ outcome = await finish(
+ route.run_stream(
+ request_body(),
+ required_labels=labels,
+ task_id=task,
+ headers={"x-switchyard-task-id": "caller-controlled"},
+ )
)
+ assert outcome.selected_model_ids[0] == expected
+ assert targets[outcome.selected_model_ids[0]] == {
+ "client": "standard" if expected == "fast" else "approved",
+ "id": f"provider/{expected}",
+ }
+
+
[email protected]("judge", ["llm", "decision"])
+async def test_approved_judge_calls_are_executed_by_the_python_host(
+ judge: Literal["llm", "decision"],
+) -> None:
+ route = AlgorithmFactory.from_toml(deployment(judge=judge)).build_configured()[
+ "switchyard/coding"
]
- assert len(steps) == 1
- assert isinstance(steps[0], Step.Done)
- assert steps[0].outcome.selected_model_ids == ["fast"]
+ targets = route.target_models()
+ stream = route.run_stream(request_body(), required_labels=["private", "customer_data"])
+ step = await anext(stream)
+ if judge == "decision":
+ assert isinstance(step, Step.CallDecision)
+ assert step.call.model == step.call.request["model"] == "decision/approved"
+ assert targets[step.call.model] == {"client": "scoring", "id": "decision/approved"}
+ choice = {
+ "type": "choice",
+ "data": {
+ "selected": "efficient",
+ "probabilities": {"efficient": 0.9, "capable": 0.1},
+ },
+ }
+ response = {"answers": {"route": {"value": choice}}}
+ else:
+ assert isinstance(step, Step.CallModel)
+ assert step.call.models == ["approved_judge"]
+ assert step.call.request["model"] == "approved_judge"
+ target = targets[step.call.models[0]]
+ assert target == {"client": "approved", "id": "provider/approved_judge"}
+ verdict = {
+ "crux": "bounded task",
+ "primary_rule": "SUP-1",
+ "capability_boundary": "supported",
+ "p_solve": 0.9,
+ }
+ message = {
+ "role": "assistant",
+ "content": [{"type": "text", "text": json.dumps(verdict)}],
+ "stop_reason": "end_turn",
+ }
+ response = LlmResponse.Agg({"model": target["id"], "outputs": [message]})
+
+ async def reply() -> object:
+ return response
+
+ await step.call.respond(reply())
+ outcome = await finish(stream)
+ assert outcome.selected_model_ids == ["both_fast", "both_capable"]
+ assert all(targets[key]["client"] == "approved" for key in outcome.selected_model_ids)