Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

chore(libsy): reorg libsy mods (#100)

Greg Clark8a6858b12f29
diff --git a/Cargo.lock b/Cargo.lock
index 0ed7ec265..b432a6d95 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -857,26 +857,13 @@ version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
-[[package]]
-name = "libsy"
-version = "0.1.0"
-dependencies = [
- "async-trait",
- "futures",
- "rand 0.8.7",
- "serde_json",
- "switchyard-protocol",
- "tokio",
- "tokio-stream",
-]
-
[[package]]
name = "libsy-examples"
version = "0.1.0"
dependencies = [
"async-trait",
"futures",
- "libsy",
+ "switchyard-libsy",
"switchyard-protocol",
"tokio",
"tokio-stream",
@@ -1629,6 +1616,19 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "switchyard-libsy"
+version = "0.1.0"
+dependencies = [
+ "async-trait",
+ "futures",
+ "rand 0.8.7",
+ "serde_json",
+ "switchyard-protocol",
+ "tokio",
+ "tokio-stream",
+]
+
[[package]]
name = "switchyard-protocol"
version = "0.1.0"
@@ -1645,7 +1645,6 @@ version = "0.1.0"
dependencies = [
"async-trait",
"futures-util",
- "libsy",
"pyo3",
"pyo3-async-runtimes",
"pythonize",
@@ -1654,6 +1653,7 @@ dependencies = [
"switchyard-components",
"switchyard-components-v2",
"switchyard-core",
+ "switchyard-libsy",
"switchyard-server",
"switchyard-translation",
"tokio",
diff --git a/Cargo.toml b/Cargo.toml
index 582ea8dfe..ec2ee2c7a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,7 +11,7 @@ members = [
"crates/switchyard-components-v2-macros",
"crates/switchyard-core",
"crates/switchyard-py",
- "crates/libsy-protocol",
+ "crates/protocol",
"crates/switchyard-server",
"crates/switchyard-skill-distillation",
"crates/switchyard-translation",
diff --git a/crates/libsy-examples/Cargo.toml b/crates/libsy-examples/Cargo.toml
index 82643f8b6..2f9ccfe55 100644
--- a/crates/libsy-examples/Cargo.toml
+++ b/crates/libsy-examples/Cargo.toml
@@ -13,8 +13,8 @@ repository.workspace = true
rust-version.workspace = true
[dependencies]
-libsy = { path = "../libsy" }
-switchyard-protocol = { path = "../libsy-protocol" }
+switchyard-libsy = { path = "../libsy" }
+switchyard-protocol = { path = "../protocol" }
async-trait = "0.1"
futures = "0.3"
tokio = { version = "1", features = ["full"] }
diff --git a/crates/libsy-examples/src/ensemble.rs b/crates/libsy-examples/src/ensemble.rs
index 102bcb525..c151ee48e 100644
--- a/crates/libsy-examples/src/ensemble.rs
+++ b/crates/libsy-examples/src/ensemble.rs
@@ -22,7 +22,7 @@ use std::sync::{Arc, Mutex};
use async_trait::async_trait;
-use libsy::{Algorithm, Context, Decision, Driver, LlmTargetSet, Request, Response};
+use switchyard_libsy::{Algorithm, Context, Decision, Driver, LlmTargetSet, Request, Response};
use switchyard_protocol::{completion_text, prompt_text, text_request};
/// Which step of the ensemble flow produced a decision.
@@ -100,8 +100,8 @@ impl EnsembleOrchAlgo {
/// exploring for `exploration_turns` before committing to the winningest
/// candidate (`0` = never commit, ensemble every request), routing among
/// `target_set`. Wrap it in an [`Arc`](std::sync::Arc) and drive it with
- /// [`run`](libsy::Algorithm::run) or
- /// [`run_stream`](libsy::Algorithm::run_stream).
+ /// [`run`](switchyard_libsy::Algorithm::run) or
+ /// [`run_stream`](switchyard_libsy::Algorithm::run_stream).
pub fn new(
candidate_models: Vec<String>,
judge_model: impl Into<String>,
@@ -395,8 +395,8 @@ impl Algorithm for EnsembleOrchAlgo {
#[cfg(test)]
mod tests {
use super::*;
- use libsy::{LlmResponse, LlmTarget, Response, RoutedLlmClient, Signals};
use std::sync::Mutex as StdMutex;
+ use switchyard_libsy::{LlmResponse, LlmTarget, Response, RoutedLlmClient, Signals};
use switchyard_protocol::text_response;
/// Mock client that answers candidate calls with `answer from {model}` and,
diff --git a/crates/libsy-examples/src/lib.rs b/crates/libsy-examples/src/lib.rs
index a3c9ff554..505dcc9f8 100644
--- a/crates/libsy-examples/src/lib.rs
+++ b/crates/libsy-examples/src/lib.rs
@@ -3,8 +3,8 @@
//! Reference algorithms for [`libsy`], kept out of the core crate but compiled and
//! tested here so they stay current. Each is a worked example of the
-//! [`Algorithm`](libsy::Algorithm) trait; the `examples/` directory has runnable agents
-//! that drive them. The core crate provides [`libsy::RandomAlgo`] for uniform random
+//! [`Algorithm`](switchyard_libsy::Algorithm) trait; the `examples/` directory has runnable agents
+//! that drive them. The core crate provides [`switchyard_libsy::algorithms::Random`] for uniform random
//! routing.
//!
//! - [`ensemble::EnsembleOrchAlgo`] — fan out to several models, judge, and commit
diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.toml
index df8303835..911169e52 100644
--- a/crates/libsy/Cargo.toml
+++ b/crates/libsy/Cargo.toml
@@ -2,7 +2,7 @@
# SPDX-License-Identifier: Apache-2.0
[package]
-name = "libsy"
+name = "switchyard-libsy"
version = "0.1.0"
description = "Switchyard library crate"
authors.workspace = true
@@ -16,7 +16,7 @@ async-trait = "0.1"
serde_json = "1"
futures = "0.3"
rand = "0.8"
-switchyard-protocol = { path = "../libsy-protocol" }
+switchyard-protocol = { path = "../protocol" }
tokio = { version = "1", features = ["full"] }
tokio-stream = "0.1"
diff --git a/crates/libsy/README.md b/crates/libsy/README.md
index 3282bc240..40a736f86 100644
--- a/crates/libsy/README.md
+++ b/crates/libsy/README.md
@@ -11,8 +11,8 @@ so it drops into a proxy, gateway, or agent runtime.
Build a target set, pick an algorithm, run a request:
```rust
-use libsy::{Algorithm, Context, RoutedLlmClient, LlmTarget, LlmTargetSet, Request};
-use libsy::LlmClassifierOrchAlgo;
+use switchyard_libsy::{Algorithm, Context, RoutedLlmClient, LlmTarget, LlmTargetSet, Request};
+use switchyard_libsy::algorithms::LlmClassifier;
use switchyard_protocol::{completion_text, text_request};
use std::sync::Arc;
@@ -20,7 +20,7 @@ use std::sync::Arc;
let client = Arc::new(MyClient { /* .. */ }) as Arc<dyn RoutedLlmClient>;
let target = |name: &str| LlmTarget { semantic_name: name.into(), llm_client: Some(client.clone()) };
-let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifierOrchAlgo::new(
+let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifier::new(
"classifier", "strong", "weak", 0.5,
LlmTargetSet::new(vec![target("classifier"), target("strong"), target("weak")]),
));
@@ -35,7 +35,7 @@ let req = Request {
let (trace, response) = algo.clone().run(Context::default(), req).await?; // calls in, trace + response out
println!("routed to {}", trace.last().unwrap().selected_model());
// `response.llm_response` is buffered or streamed; fold it to the aggregate to read text.
-println!("answer: {}", completion_text(&response.llm_response.aggregate().await?));
+println!("answer: {}", completion_text(&response.llm_response.into_agg().await?));
```
Runnable: [`research_agent`](examples/research_agent.rs)
@@ -152,17 +152,17 @@ match response.llm_response {
}
```
-Need the whole answer regardless of how it arrived? `aggregate()` folds a `Stream` (or
+Need the whole answer regardless of how it arrived? `into_agg()` folds a `Stream` (or
returns an `Agg` unchanged) into a single `AggLlmResponse`, surfacing any mid-stream error:
```rust
-let agg = response.llm_response.aggregate().await?; // drives the stream to completion
+let agg = response.llm_response.into_agg().await?; // drives the stream to completion
println!("{}", completion_text(&agg));
```
An `LlmResponseChunk` is the provider-neutral streaming event (`MessageStart`, `TextDelta`,
`ReasoningDelta`, `ToolCallDelta`, `Usage`, `MessageStop`, `Error`); `ResponseAccumulator`
-is the fold behind `aggregate()` if you need to assemble one yourself. Runnable end-to-end
+is the fold behind `into_agg()` if you need to assemble one yourself. Runnable end-to-end
demo: [`streaming_agent`](examples/streaming_agent.rs).
## Driving the calls yourself (`run_stream`)
@@ -221,7 +221,7 @@ Example — the LLM classifier (classify, then route; full version in
```rust
#[async_trait]
-impl Algorithm for LlmClassifierOrchAlgo {
+impl Algorithm for LlmClassifier {
async fn create_run_task(self: Arc<Self>, ctx: Context, driver: Driver, request: Request)
-> Result<Response, Box<dyn Error + Send + Sync>> {
// Thread `ctx` into every offloaded call and decision — it carries the request's
@@ -233,8 +233,8 @@ impl Algorithm for LlmClassifierOrchAlgo {
let classify_response =
driver.call_llm_target(ctx.clone(), &classifier, classify_req, classify_decision).await?;
// Abbreviated: fold the (buffered or streamed) response to its aggregate and read
- // the completion text as a score. `.agg()` alone is `None` for an unfolded stream.
- let score = classify_response.llm_response.aggregate().await
+ // the completion text as a score. `.as_agg()` alone is `None` for an unfolded stream.
+ let score = classify_response.llm_response.into_agg().await
.ok()
.and_then(|agg| completion_text(&agg).trim().parse::<f64>().ok());
@@ -256,9 +256,9 @@ agents live in [`examples`](examples/) folder.
**Reference algorithms** — implementations to read and route with:
-- [`RandomAlgo`](src/algorithms/rand.rs) — uniform random over the set
+- [`Random`](src/algorithms/rand.rs) — uniform random over the set
(one call).
-- [`LlmClassifierOrchAlgo`](src/algorithms/llm_class.rs) — classify, then route
+- [`LlmClassifier`](src/algorithms/llm_class.rs) — classify, then route
strong/weak; fail open to strong.
- [`EnsembleOrchAlgo`](../libsy-examples/src/ensemble.rs) — stateful: fan out to
candidates, judge the best, commit to the winner after N exploration turns.
diff --git a/crates/libsy/examples/research_agent.rs b/crates/libsy/examples/research_agent.rs
index eabfb9bda..f3fd2f873 100644
--- a/crates/libsy/examples/research_agent.rs
+++ b/crates/libsy/examples/research_agent.rs
@@ -15,8 +15,8 @@ use std::error::Error;
use std::sync::Arc;
use async_trait::async_trait;
-use libsy::LlmClassifierOrchAlgo;
-use libsy::{
+use switchyard_libsy::algorithms::LlmClassifier;
+use switchyard_libsy::{
Algorithm, Context, Decision, LlmResponse, LlmTarget, LlmTargetSet, Request, Response,
RoutedLlmClient,
};
@@ -91,14 +91,9 @@ impl ResearchAgent {
#[tokio::main(flavor = "current_thread")]
async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
// Configure routing once: an LLM classifier over three named targets. Swapping
- // in `RandomAlgo` needs no change to the agent.
- let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifierOrchAlgo::new(
- CLASSIFIER,
- STRONG,
- WEAK,
- 0.5,
- targets(),
- ));
+ // in `Random` needs no change to the agent.
+ let algo: Arc<dyn Algorithm> =
+ Arc::new(LlmClassifier::new(CLASSIFIER, STRONG, WEAK, 0.5, targets()));
let agent = ResearchAgent { algo };
println!("{}", agent.run("what is switchyard?").await?);
diff --git a/crates/libsy/examples/research_agent_core.rs b/crates/libsy/examples/research_agent_core.rs
index 0199728bb..64e2dbac8 100644
--- a/crates/libsy/examples/research_agent_core.rs
+++ b/crates/libsy/examples/research_agent_core.rs
@@ -12,8 +12,8 @@
use std::error::Error;
use std::sync::Arc;
-use libsy::LlmClassifierOrchAlgo;
-use libsy::{
+use switchyard_libsy::algorithms::LlmClassifier;
+use switchyard_libsy::{
Algorithm, Context, Decision, LlmResponse, LlmTarget, LlmTargetSet, Request, Response, Step,
};
use switchyard_protocol::{completion_text, text_request, text_response};
@@ -98,13 +98,8 @@ fn print_decision(decision: &dyn Decision) {
#[tokio::main(flavor = "current_thread")]
async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
- let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifierOrchAlgo::new(
- CLASSIFIER,
- STRONG,
- WEAK,
- 0.5,
- targets(),
- ));
+ let algo: Arc<dyn Algorithm> =
+ Arc::new(LlmClassifier::new(CLASSIFIER, STRONG, WEAK, 0.5, targets()));
let mut agent = ResearchAgent { algo };
println!("{}", agent.run("what is switchyard?").await?);
diff --git a/crates/libsy/examples/streaming_agent.rs b/crates/libsy/examples/streaming_agent.rs
index 5c245a8b9..7038a33e2 100644
--- a/crates/libsy/examples/streaming_agent.rs
+++ b/crates/libsy/examples/streaming_agent.rs
@@ -17,9 +17,10 @@ use std::io::Write;
use std::sync::Arc;
use futures::StreamExt;
-use libsy::{
+use switchyard_libsy::algorithms::Random;
+use switchyard_libsy::{
Algorithm, Context, LlmResponse, LlmResponseChunk, LlmResponseStream, LlmTarget, LlmTargetSet,
- RandomAlgo, Request, Response, Step,
+ Request, Response, Step,
};
use switchyard_protocol::{completion_text, text_request};
@@ -55,7 +56,7 @@ async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
semantic_name: "stream/model".to_string(),
llm_client: None,
}]);
- let algo: Arc<dyn Algorithm> = Arc::new(RandomAlgo::new(targets));
+ let algo: Arc<dyn Algorithm> = Arc::new(Random::new(targets));
let request = Request {
llm_request: text_request(Some("auto".to_string()), "tell me about switchyard"),
diff --git a/crates/libsy/src/algorithms.rs b/crates/libsy/src/algorithms.rs
index ce17da02b..6288e4aec 100644
--- a/crates/libsy/src/algorithms.rs
+++ b/crates/libsy/src/algorithms.rs
@@ -1,6 +1,15 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
+//! Concrete algorithms and the interfaces for building them.
+//!
+//! Reach for them by name — `use switchyard_libsy::algorithms::Random` — rather than through the
+//! per-algorithm submodules.
+
pub mod llm_class;
pub mod noop;
pub mod rand;
+
+pub use llm_class::{ClassifierDecision, ClassifierTier, LlmClassifier};
+pub use noop::{Noop, NoopDecision};
+pub use rand::{Random, RandomDecision};
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index e3fe7ebf0..aa10a9fef 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -64,7 +64,7 @@ impl Decision for ClassifierDecision {
}
/// LLM-classifier router: classify with one target, then route to strong/weak.
-pub struct LlmClassifierOrchAlgo {
+pub struct LlmClassifier {
classifier_model: String,
strong_model: String,
weak_model: String,
@@ -72,7 +72,7 @@ pub struct LlmClassifierOrchAlgo {
target_set: LlmTargetSet,
}
-impl LlmClassifierOrchAlgo {
+impl LlmClassifier {
/// Configure the classifier: the model that scores each request, the strong
/// and weak models to route to, the score `threshold` at or above which the
/// strong model is chosen, and the `target_set` to route among. That set must
@@ -138,7 +138,7 @@ fn trim_messages(messages: &[Message], recent_turn_window: usize) -> Vec<Message
}
#[async_trait]
-impl Algorithm for LlmClassifierOrchAlgo {
+impl Algorithm for LlmClassifier {
async fn create_run_task(
self: Arc<Self>,
ctx: Context,
@@ -251,7 +251,7 @@ mod tests {
}
/// Build a classifier algo whose three targets share a scoring client.
- fn algo(threshold: f64, score: &str) -> (LlmClassifierOrchAlgo, Arc<Mutex<Vec<Request>>>) {
+ fn algo(threshold: f64, score: &str) -> (LlmClassifier, Arc<Mutex<Vec<Request>>>) {
let seen = Arc::new(Mutex::new(Vec::new()));
let client = Arc::new(ScoringClient {
classifier_model: "router/classifier".to_string(),
@@ -267,7 +267,7 @@ mod tests {
target("frontier/model"),
target("cheap/model"),
]);
- let algo = LlmClassifierOrchAlgo {
+ let algo = LlmClassifier {
classifier_model: "router/classifier".to_string(),
strong_model: "frontier/model".to_string(),
weak_model: "cheap/model".to_string(),
@@ -286,7 +286,7 @@ mod tests {
}
/// Wrap a classifier algo as `Arc<dyn Algorithm>` we can drive to completion.
- fn orch(algo: LlmClassifierOrchAlgo) -> Arc<dyn Algorithm> {
+ fn orch(algo: LlmClassifier) -> Arc<dyn Algorithm> {
Arc::new(algo)
}
diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rs
index cefab1165..a3ec368cd 100644
--- a/crates/libsy/src/algorithms/noop.rs
+++ b/crates/libsy/src/algorithms/noop.rs
@@ -13,9 +13,9 @@ use switchyard_protocol::{
use crate::{Algorithm, Context, Decision, Driver};
/// A routing algorithm that does not route. It returns a hard-coded response.
-pub struct NoopAlgo {}
+pub struct Noop {}
-/// How [`NoopAlgo`] records which model it chose. This will be the model on the Request if any,
+/// How [`Noop`] records which model it chose. This will be the model on the Request if any,
/// otherwise a hard coded placeholder. Neither is actually used.
pub struct NoopDecision {
model: String,
@@ -34,7 +34,7 @@ impl Decision for NoopDecision {
}
#[async_trait::async_trait]
-impl Algorithm for NoopAlgo {
+impl Algorithm for Noop {
async fn create_run_task(
self: Arc<Self>,
ctx: Context,
@@ -89,7 +89,7 @@ mod tests {
metadata: None,
};
- let a: Arc<dyn Algorithm> = Arc::new(NoopAlgo {});
+ let a: Arc<dyn Algorithm> = Arc::new(Noop {});
let (decisions, response) = a.run(Context::default(), request).await?;
let Some(decision) = decisions.first() else {
panic!("Expected exactly one Decision");
diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rs
index adc2b366e..817fa9566 100644
--- a/crates/libsy/src/algorithms/rand.rs
+++ b/crates/libsy/src/algorithms/rand.rs
@@ -16,7 +16,7 @@ use rand::seq::SliceRandom;
use crate::{Algorithm, Context, Decision, Driver, LlmTargetSet, Request, Response};
-/// Decision produced by [`RandomAlgo`]: which target was chosen and why.
+/// Decision produced by [`Random`]: which target was chosen and why.
pub struct RandomDecision {
/// The randomly selected target/model.
pub selected_model: String,
@@ -39,11 +39,11 @@ impl Decision for RandomDecision {
}
/// Uniform random router over a target set.
-pub struct RandomAlgo {
+pub struct Random {
target_set: LlmTargetSet,
}
-impl RandomAlgo {
+impl Random {
/// Creates a router over `target_set`.
///
/// Wrap it in an [`Arc`] and drive it with [`Algorithm::run`] or
@@ -54,7 +54,7 @@ impl RandomAlgo {
}
#[async_trait]
-impl Algorithm for RandomAlgo {
+impl Algorithm for Random {
async fn create_run_task(
self: Arc<Self>,
ctx: Context,
@@ -120,7 +120,7 @@ mod tests {
}
/// Builds a random router whose targets all share an echo client.
- fn algorithm(names: &[&str]) -> RandomAlgo {
+ fn algorithm(names: &[&str]) -> Random {
let targets = names
.iter()
.map(|name| LlmTarget {
@@ -128,7 +128,7 @@ mod tests {
llm_client: Some(Arc::new(EchoClient)),
})
.collect();
- RandomAlgo::new(LlmTargetSet::new(targets))
+ Random::new(LlmTargetSet::new(targets))
}
fn shared_algorithm(names: &[&str]) -> Arc<dyn Algorithm> {
diff --git a/crates/libsy/src/core.rs b/crates/libsy/src/core.rs
new file mode 100644
index 000000000..a46cf2f4c
--- /dev/null
+++ b/crates/libsy/src/core.rs
@@ -0,0 +1,12 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Core orchestration: the [`Algorithm`] trait and its [`Driver`], built on the
+//! type-erased promise-over-a-stream pump in [`driver`]. Its public items are
+//! re-exported at the crate root; algorithm implementations live in
+//! [`crate::algorithms`].
+
+pub mod algorithm;
+mod driver;
+
+pub use algorithm::*;
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
new file mode 100644
index 000000000..621fb980b
--- /dev/null
+++ b/crates/libsy/src/core/algorithm.rs
@@ -0,0 +1,931 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! The [`Algorithm`] trait and its [`Driver`] — the orchestration contract every
+//! routing/optimization algorithm implements, and the offload channel it makes model
+//! calls and publishes [`Decision`]s over. See the crate root for the narrative model.
+
+use std::{error::Error, pin::Pin, sync::Arc};
+
+use async_trait::async_trait;
+use futures::{Stream, StreamExt};
+
+/// The request/response protocol types, re-exported from [`switchyard_protocol`].
+/// [`LlmRequest`] is the normalized request; [`AggLlmResponse`] is the buffered response;
+/// [`LlmResponseChunk`] is one streaming event; [`LlmResponse`] is the streamed response
+/// (a live [`LlmResponseStream`] or the terminal aggregate).
+pub use switchyard_protocol::{
+ AggLlmResponse, Context, Decision, LlmRequest, LlmResponse, LlmResponseChunk,
+ LlmResponseStream, Metadata, Request, Response, RoutedLlmClient, Signals,
+};
+
+use super::driver::{DriverRequest, DriverStep, TypeErasedDriver};
+
+/// Shorthand for the crate's boxed, thread-safe error type.
+type BoxErr = Box<dyn Error + Send + Sync>;
+
+/// A boxed, `Send` stream of [`Step`]s — the output of
+/// [`Algorithm::run_stream`]. Boxed so the trait method that produces it keeps
+/// `Arc<dyn Algorithm>` object-safe.
+pub type StepStream = Pin<Box<dyn Stream<Item = Result<Step, BoxErr>> + Send>>;
+
+/// A request paired with the routing [`Decision`] that produced it — the offload
+/// payload a host reads (via [`CallLlmRequest::get_routed`]) to serve the call.
+///
+/// The two model identifiers live in separate, unambiguous places: the model to
+/// call is [`decision.selected_model()`](Decision::selected_model), while
+/// `request.llm_request.model` is the *inbound* name the agent asked for (libsy
+/// never overwrites it). A client maps `selected_model()` to the provider model
+/// id it hits.
+#[derive(Clone)]
+pub struct RoutedRequest {
+ /// The request to serve; its `model` is the agent's original name.
+ pub request: Request,
+ /// The routing decision behind this call; `selected_model()` is the model to hit.
+ pub decision: Arc<dyn Decision>,
+ /// The client that serves this call by default, or `None` when the routed target
+ /// had no client. Rides along on the offloaded call so a host driving the stream
+ /// can serve it by default or override it with its own transport.
+ pub default_client: Option<Arc<dyn RoutedLlmClient>>,
+ /// The request's cross-cutting context, carried through the offload so whoever
+ /// serves the call (libsy's own `run`, or a host driving the stream) hands it to
+ /// [`RoutedLlmClient::call`].
+ pub ctx: Context,
+}
+
+/// The host-facing half of an offloaded model call, surfaced inside [`Step::CallLlm`].
+///
+/// Wraps a [`DriverRequest`] whose payload is a [`RoutedRequest`]. The host reads the
+/// routed request ([`get_routed`](Self::get_routed)) and the decision behind it
+/// ([`get_decision`](Self::get_decision)), performs (or delegates) the model call, and
+/// fulfills it with [`respond`](Self::respond) — unblocking the algorithm's
+/// [`Driver::call_llm`] on the other side.
+pub struct CallLlmRequest {
+ inner: DriverRequest,
+ routed: RoutedRequest,
+}
+
+impl CallLlmRequest {
+ /// Wrap a driver request whose payload is a [`RoutedRequest`]. Caches an owned copy
+ /// so the accessors are plain field reads.
+ fn new(inner: DriverRequest) -> Self {
+ // The payload is always a `RoutedRequest` (set by `Driver::call_llm`); a
+ // mismatch would be a libsy bug, not a runtime condition.
+ let routed = match inner.request::<RoutedRequest>() {
+ Ok(routed) => routed.clone(),
+ Err(_) => unreachable!("CallLlmRequest payload is always a RoutedRequest"),
+ };
+ Self { inner, routed }
+ }
+
+ /// The routed request the host should serve. Its
+ /// [`default_client`](RoutedRequest::default_client) serves the call by default,
+ /// and its `decision.selected_model()` names the model to hit.
+ pub fn get_routed(&self) -> &RoutedRequest {
+ &self.routed
+ }
+
+ /// The model request to perform (the [`Request`] inside the routed request).
+ pub fn get_request(&self) -> &Request {
+ &self.get_routed().request
+ }
+
+ /// The decision that led to this call — its `selected_model()` is the model to hit.
+ pub fn get_decision(&self) -> &dyn Decision {
+ self.get_routed().decision.as_ref()
+ }
+
+ /// Fulfill the promise with the caller's model-call result. Pass `Err(..)` to
+ /// propagate a failed model call back to the algorithm. Consumes the promise: it
+ /// can only be fulfilled once.
+ pub fn respond(self, result: Result<Response, BoxErr>) -> Result<(), BoxErr> {
+ self.inner.respond::<Response>(result)
+ }
+}
+
+/// The offload channel handed to an algorithm's
+/// [`create_run_task`](Algorithm::create_run_task). The algorithm makes model calls
+/// with [`call_llm_target`](Self::call_llm_target) (or [`call_llm`](Self::call_llm)) and
+/// publishes its [`Decision`]s with [`info`](Self::info); each call is offloaded to the
+/// request's [`Step`] stream and awaits the consumer's response. The step channel is
+/// bounded, so the consumer paces the algorithm one step at a time.
+#[derive(Clone)]
+pub struct Driver {
+ driver: TypeErasedDriver,
+}
+
+impl Driver {
+ /// Build an empty driver with its step channel ready. Created per call by
+ /// [`run_stream`](Algorithm::run_stream).
+ pub(crate) fn new() -> Self {
+ Self {
+ driver: TypeErasedDriver::new(),
+ }
+ }
+
+ /// Offload a model call: publish `routed` as a [`Step::CallLlm`] and await the
+ /// consumer's [`Response`]. The call's context travels inside
+ /// [`routed.ctx`](RoutedRequest::ctx). Errors if the stream is closed or the call failed.
+ pub async fn call_llm(&self, routed: RoutedRequest) -> Result<Response, BoxErr> {
+ self.driver
+ .fulfill_request::<RoutedRequest, Response>(routed.ctx.clone(), routed)
+ .await
+ }
+
+ /// Offload a call to `target`: pair `request` with `decision` and the target's
+ /// default client into a [`RoutedRequest`], then publish it (see
+ /// [`call_llm`](Self::call_llm)). The convenience most algorithms use;
+ /// `decision.selected_model()` names the model to hit, and `request`'s
+ /// `model` is left untouched.
+ pub async fn call_llm_target(
+ &self,
+ ctx: Context,
+ target: &LlmTarget,
+ request: Request,
+ decision: Arc<dyn Decision>,
+ ) -> Result<Response, BoxErr> {
+ self.call_llm(RoutedRequest {
+ request,
+ decision,
+ default_client: target.llm_client.clone(),
+ ctx,
+ })
+ .await
+ }
+
+ /// Publish a routing [`Decision`] as a [`Step::Decision`] on the stream.
+ pub async fn info(&self, ctx: Context, decision: Arc<dyn Decision>) -> Result<(), BoxErr> {
+ self.driver.info(ctx, decision).await
+ }
+
+ /// Emit the terminal step: [`Step::ReturnToAgent`] on `Ok`, or an `Err` stream
+ /// item on failure. Internal: called once by [`run_stream`](Algorithm::run_stream)
+ /// when the algorithm finishes.
+ pub(crate) async fn finish(
+ &self,
+ ctx: Context,
+ result: Result<Response, BoxErr>,
+ ) -> Result<(), BoxErr> {
+ match result {
+ Ok(response) => self.driver.done(ctx, response).await,
+ Err(err) => self.driver.fail(ctx, err).await,
+ }
+ }
+
+ /// Transform the raw driver stream into a stream of [`Step`]s. Internal: the
+ /// consumer stream is taken (once) by [`run_stream`](Algorithm::run_stream). A
+ /// payload that does not match the expected type for its step becomes an `Err` item.
+ pub(crate) fn stream(&self) -> impl Stream<Item = Result<Step, BoxErr>> {
+ self.driver.stream().map(|item| match item? {
+ DriverStep::Request(req) => Ok(Step::CallLlm(Box::new(CallLlmRequest::new(req)))),
+ DriverStep::Info(payload) => payload
+ .downcast::<Arc<dyn Decision>>()
+ .map(|decision| Step::Decision(*decision))
+ .map_err(|_| "driver: info payload was not a Decision".into()),
+ DriverStep::Done(payload) => payload
+ .downcast::<Response>()
+ .map(Step::ReturnToAgent)
+ .map_err(|_| "driver: done payload was not a Response".into()),
+ })
+ }
+}
+
+impl Default for Driver {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+/// One item in the stream returned by [`Driver::stream`] / [`Algorithm::run_stream`].
+pub enum Step {
+ /// The algorithm needs this model call performed. The host serves it (optionally
+ /// via [`RoutedRequest::default_client`]) and fulfills it with
+ /// [`CallLlmRequest::respond`]. Boxed: it is by far the largest variant.
+ CallLlm(Box<CallLlmRequest>),
+ /// A routing decision the algorithm made, published via [`Driver::info`] as it
+ /// happens (rather than collected into a trace returned at the end).
+ Decision(Arc<dyn Decision>),
+ /// The algorithm finished with its final response — the last step of a run.
+ ReturnToAgent(Box<Response>),
+}
+
+/// A named routing target: a `semantic_name` an algorithm routes by, and an optional
+/// [`RoutedLlmClient`] to serve its calls. An algorithm hands a target to
+/// [`Driver::call_llm_target`]; the client rides along as
+/// [`RoutedRequest::default_client`] for the stream consumer to serve or override.
+#[derive(Clone)]
+pub struct LlmTarget {
+ /// The routing label an algorithm selects this target by — a logical tier like
+ /// `"strong"`, or the model id when they coincide. Mapping it to a provider model
+ /// id is the client's concern, never the algorithm's.
+ pub semantic_name: String,
+ /// The client that serves this target's calls by default, or `None` (then the
+ /// stream consumer must serve them).
+ pub llm_client: Option<Arc<dyn RoutedLlmClient>>,
+}
+
+/// The set of targets an algorithm may route among. An algorithm is constructed
+/// with one and picks targets by position ([`targets`](Self::targets)) or by name
+/// ([`get_target`](Self::get_target)).
+#[derive(Clone)]
+pub struct LlmTargetSet {
+ targets: Vec<LlmTarget>,
+}
+
+impl LlmTargetSet {
+ /// Build a target set from a list of targets.
+ pub fn new(targets: Vec<LlmTarget>) -> Self {
+ Self { targets }
+ }
+
+ /// All targets in the set — e.g. for an algorithm to select among.
+ pub fn targets(&self) -> &[LlmTarget] {
+ &self.targets
+ }
+
+ /// Look up a target by name; errors if no target has that name.
+ pub fn get_target(&self, name: &str) -> Result<LlmTarget, Box<dyn Error + Send + Sync>> {
+ self.targets
+ .iter()
+ .find(|t| t.semantic_name == name)
+ .cloned()
+ .ok_or(format!("Target {} not found", name).into())
+ }
+}
+
+/// An optimization strategy. Implement [`create_run_task`](Self::create_run_task);
+/// callers drive it with the provided [`run`](Self::run) (serve calls, get the answer)
+/// or [`run_stream`](Self::run_stream) (drive the [`Step`] stream yourself).
+///
+/// Methods take `self: Arc<Self>`: one algorithm (`Arc<dyn Algorithm>`) is shared across
+/// requests and run concurrently, so it owns its thread-safety. Stateless algorithms
+/// (the reference routers) get this for free; a stateful one uses interior mutability
+/// over just its own state.
+#[async_trait]
+pub trait Algorithm: Send + Sync + 'static {
+ /// Run one request to completion: make model calls with [`Driver::call_llm_target`],
+ /// publish [`Decision`]s with [`Driver::info`], and return the final [`Response`].
+ /// The method an algorithm implements; [`run`](Self::run) / [`run_stream`](Self::run_stream)
+ /// drive it. `ctx` carries cross-cutting request state (empty today).
+ async fn create_run_task(
+ self: Arc<Self>,
+ ctx: Context,
+ driver: Driver,
+ request: Request,
+ ) -> Result<Response, Box<dyn Error + Send + Sync>>;
+
+ /// Feed the algorithm agentic-stack events (tool results, budgets, etc.). The
+ /// reference algorithms ignore signals; a stateful algorithm updates its own
+ /// (interior-mutable) state. Takes `self: Arc<Self>` like the other run methods.
+ #[allow(unused_variables)]
+ async fn process_signals(
+ self: Arc<Self>,
+ signals: Signals,
+ ) -> Result<(), Box<dyn Error + Send + Sync>> {
+ Ok(())
+ }
+
+ /// Run one request as a stream of [`Step`]s (provided). The algorithm runs on its
+ /// own task; drive the stream: serve each [`Step::CallLlm`] (via its
+ /// [`default_client`](RoutedRequest::default_client) or your own transport) and read
+ /// [`Step::Decision`]s until the final [`Step::ReturnToAgent`]. The step channel is
+ /// bounded, so pulling paces the algorithm; each call is independent, so many run
+ /// concurrently.
+ fn run_stream(self: Arc<Self>, ctx: Context, request: Request) -> StepStream {
+ // This call's own driver: take its consumer stream, hand a producer-side clone to
+ // the algorithm task, and keep one to emit the terminal step. The task blocks
+ // publishing a step until the consumer pulls the previous one.
+ let driver = Driver::new();
+ let stream = driver.stream();
+ tokio::spawn(async move {
+ let outcome = self
+ .create_run_task(ctx.clone(), driver.clone(), request)
+ .await;
+ let _ = driver.finish(ctx, outcome).await;
+ });
+ Box::pin(stream)
+ }
+
+ /// Run one request to completion, serving each offloaded call with its
+ /// [`RoutedRequest::default_client`], and return the decision trace plus the final
+ /// [`Response`]. Provided: drives [`run_stream`](Self::run_stream) internally,
+ /// collecting each [`Step::Decision`]. Use it when the algorithm holds its own model
+ /// clients and the host wants the answer (and the decisions behind it); drive
+ /// [`run_stream`](Self::run_stream) instead to serve the calls yourself. The returned
+ /// [`Response`] is whatever the algorithm produced — a buffered [`LlmResponse::Agg`]
+ /// or a live [`LlmResponse::Stream`]; the caller decides whether to
+ /// [`into_agg`](LlmResponse::into_agg) it or forward its chunks. Errors if a routed
+ /// target has no client to serve its call, or the algorithm fails.
+ async fn run(
+ self: Arc<Self>,
+ ctx: Context,
+ request: Request,
+ ) -> Result<(Vec<Arc<dyn Decision>>, Response), Box<dyn Error + Send + Sync>> {
+ // Serve up to this many offloaded calls concurrently, so an algorithm that
+ // fans out (e.g. an ensemble) isn't serialized on the client.
+ const MAX_CONCURRENT_CALLS: usize = 10;
+
+ // Serve one offloaded call with its target's default client. A failed *model*
+ // call is forwarded to the algorithm via `respond`; this errors only on an
+ // infrastructure failure (no default client, or the promise was dropped).
+ async fn serve(call: CallLlmRequest) -> Result<(), Box<dyn Error + Send + Sync>> {
+ let routed = call.get_routed().clone();
+ let client = routed.default_client.clone().ok_or_else(|| {
+ format!(
+ "run: target '{}' has no client to serve the call",
+ routed.decision.selected_model()
+ )
+ })?;
+ call.respond(
+ client
+ .call(routed.ctx, routed.request, routed.decision)
+ .await,
+ )
+ }
+
+ let stream = self.run_stream(ctx, request);
+ tokio::pin!(stream);
+
+ let mut trace: Vec<Arc<dyn Decision>> = Vec::new();
+ let mut in_flight = futures::stream::FuturesUnordered::new();
+ let mut final_response: Option<Response> = None;
+ let mut stream_open = true;
+
+ while stream_open || !in_flight.is_empty() {
+ tokio::select! {
+ // Surface a failed serve as soon as it completes.
+ Some(result) = in_flight.next(), if !in_flight.is_empty() => result?,
+ // Pull the next step, unless the stream ended or we're at the cap.
+ step = stream.next(), if stream_open && in_flight.len() < MAX_CONCURRENT_CALLS => {
+ match step {
+ None => stream_open = false,
+ Some(item) => match item? {
+ Step::CallLlm(call) => in_flight.push(serve(*call)),
+ Step::Decision(decision) => trace.push(decision),
+ Step::ReturnToAgent(response) => {
+ final_response = Some(*response);
+ stream_open = false;
+ }
+ },
+ }
+ }
+ }
+ }
+
+ // Return whatever the algorithm produced — buffered or streamed. The caller
+ // decides whether to aggregate the response or forward its chunks.
+ final_response
+ .map(|response| (trace, response))
+ .ok_or_else(|| "run: stream ended without a final response".into())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use futures::StreamExt;
+ use switchyard_protocol::{completion_text, text_request, text_response};
+
+ /// Mock client that echoes back the target name it was called with.
+ struct EchoClient;
+
+ #[async_trait]
+ impl RoutedLlmClient for EchoClient {
+ async fn call(
+ &self,
+ _ctx: Context,
+ _request: Request,
+ decision: Arc<dyn Decision>,
+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {
+ // Echo back the model the algorithm routed to (the decision's selection).
+ Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(
+ None,
+ decision.selected_model().to_string(),
+ )),
+ metadata: None,
+ })
+ }
+ }
+
+ /// Trivial decision + algo used only to exercise the orchestrator: calls the
+ /// first target and returns its response with a one-item trace.
+ struct TestDecision {
+ model: String,
+ }
+
+ impl Decision for TestDecision {
+ fn selected_model(&self) -> &str {
+ &self.model
+ }
+ fn reasoning(&self) -> Option<&str> {
+ None
+ }
+ fn as_any(&self) -> &dyn std::any::Any {
+ self
+ }
+ }
+
+ struct TestAlgo {
+ target_set: LlmTargetSet,
+ }
+
+ #[async_trait]
+ impl Algorithm for TestAlgo {
+ async fn create_run_task(
+ self: Arc<Self>,
+ ctx: Context,
+ driver: Driver,
+ request: Request,
+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {
+ let target = self
+ .target_set
+ .targets()
+ .first()
+ .ok_or("no targets")?
+ .clone();
+ let decision: Arc<dyn Decision> = Arc::new(TestDecision {
+ model: target.semantic_name.clone(),
+ });
+ driver.info(ctx.clone(), decision.clone()).await?;
+ driver
+ .call_llm_target(ctx, &target, request, decision)
+ .await
+ }
+ }
+
+ /// Build a shared `TestAlgo` over the given target set.
+ fn orch(target_set: LlmTargetSet) -> Arc<dyn Algorithm> {
+ Arc::new(TestAlgo { target_set })
+ }
+
+ fn request() -> Request {
+ Request {
+ llm_request: text_request(Some("auto".to_string()), "hi".to_string()),
+ raw_request: None,
+ metadata: None,
+ }
+ }
+
+ /// `(name, has_client)` — `has_client: false` builds a target with no default client.
+ fn target_set(names: &[(&str, bool)]) -> LlmTargetSet {
+ let targets = names
+ .iter()
+ .map(|(name, has_client)| LlmTarget {
+ semantic_name: name.to_string(),
+ llm_client: has_client.then(|| Arc::new(EchoClient) as Arc<dyn RoutedLlmClient>),
+ })
+ .collect();
+ LlmTargetSet::new(targets)
+ }
+
+ /// Client that serves a call as a token stream — its `call` returns
+ /// [`LlmResponse::Stream`] replaying `chunks` in order (as `Ok` items).
+ struct StreamingClient {
+ chunks: Vec<LlmResponseChunk>,
+ }
+
+ #[async_trait]
+ impl RoutedLlmClient for StreamingClient {
+ async fn call(
+ &self,
+ _ctx: Context,
+ _request: Request,
+ _decision: Arc<dyn Decision>,
+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {
+ let stream = futures::stream::iter(self.chunks.clone().into_iter().map(Ok)).boxed();
+ Ok(Response {
+ llm_response: LlmResponse::Stream(stream),
+ metadata: None,
+ })
+ }
+ }
+
+ /// Build a single-target algo whose one target streams `chunks`.
+ fn streaming_orch(chunks: Vec<LlmResponseChunk>) -> Arc<dyn Algorithm> {
+ let target = LlmTarget {
+ semantic_name: "stream/model".to_string(),
+ llm_client: Some(Arc::new(StreamingClient { chunks }) as Arc<dyn RoutedLlmClient>),
+ };
+ orch(LlmTargetSet::new(vec![target]))
+ }
+
+ #[tokio::test]
+ async fn run_returns_a_streamed_response_the_caller_aggregates(
+ ) -> Result<(), Box<dyn Error + Send + Sync>> {
+ // A streaming client -> its chunks flow through the promise and `ReturnToAgent`,
+ // and `run` returns the live stream untouched for the caller to fold.
+ let orch = streaming_orch(vec![
+ LlmResponseChunk::MessageStart {
+ id: Some("m1".to_string()),
+ model: Some("stream/model".to_string()),
+ },
+ LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "hel".to_string(),
+ },
+ LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "lo".to_string(),
+ },
+ LlmResponseChunk::MessageStop {
+ reason: Some("stop".to_string()),
+ },
+ ]);
+ let (trace, response) = orch.run(Context::default(), request()).await?;
+ // `run` handed back the live stream; the caller folds it to a buffered aggregate.
+ let agg = response.llm_response.into_agg().await?;
+ assert_eq!(completion_text(&agg), "hello");
+ assert_eq!(agg.model.as_deref(), Some("stream/model"));
+ assert_eq!(trace.len(), 1);
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn aggregating_a_streamed_response_propagates_a_mid_stream_error(
+ ) -> Result<(), Box<dyn Error + Send + Sync>> {
+ // `run` succeeds and returns the stream; the in-band `Error` chunk surfaces only
+ // when the caller aggregates it.
+ let orch = streaming_orch(vec![
+ LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "partial".to_string(),
+ },
+ LlmResponseChunk::Error {
+ message: "upstream exploded".to_string(),
+ },
+ ]);
+ let (_, response) = orch.run(Context::default(), request()).await?;
+ match response.llm_response.into_agg().await {
+ Ok(_) => Err("expected a mid-stream error, got an aggregate".into()),
+ Err(err) => {
+ assert!(err.to_string().contains("upstream exploded"));
+ Ok(())
+ }
+ }
+ }
+
+ #[tokio::test]
+ async fn run_offloads_via_promise_then_returns_to_agent(
+ ) -> Result<(), Box<dyn Error + Send + Sync>> {
+ // A client-less target -> its call is offloaded via a promise the
+ // orchestrator surfaces as a `CallLlm` step for us to fulfill.
+ let stream =
+ orch(target_set(&[("offload/model", false)])).run_stream(Context::default(), request());
+ tokio::pin!(stream);
+
+ let mut saw_call = false;
+ let mut final_completion = None;
+ while let Some(step) = stream.next().await {
+ match step? {
+ Step::CallLlm(call) => {
+ saw_call = true;
+ // The decision rode along with the promise.
+ assert_eq!(call.get_decision().selected_model(), "offload/model");
+ // Fulfilling the promise is the "real" model call the caller makes.
+ call.respond(Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(
+ None,
+ "fulfilled".to_string(),
+ )),
+ metadata: None,
+ }))?;
+ }
+ Step::Decision(decision) => {
+ assert_eq!(decision.selected_model(), "offload/model");
+ }
+ Step::ReturnToAgent(response) => {
+ final_completion = Some(
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ );
+ }
+ }
+ }
+
+ assert!(saw_call, "expected a CallLlm step before ReturnToAgent");
+ assert_eq!(
+ final_completion.ok_or("no ReturnToAgent step")?,
+ "fulfilled"
+ );
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn client_backed_target_offloads_with_a_default_client(
+ ) -> Result<(), Box<dyn Error + Send + Sync>> {
+ // Every call now offloads to the stream; a client-backed target rides its
+ // client along as `default_client` so the consumer can serve it by default.
+ let stream =
+ orch(target_set(&[("direct/model", true)])).run_stream(Context::default(), request());
+ tokio::pin!(stream);
+
+ let mut final_completion = None;
+ while let Some(step) = stream.next().await {
+ match step? {
+ Step::CallLlm(call) => {
+ let routed = call.get_routed().clone();
+ let client = routed
+ .default_client
+ .clone()
+ .ok_or("expected a default client")?;
+ let result = client
+ .call(routed.ctx, routed.request, routed.decision)
+ .await;
+ call.respond(result)?;
+ }
+ Step::Decision(_) => {}
+ Step::ReturnToAgent(response) => {
+ final_completion = Some(
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ );
+ }
+ }
+ }
+
+ // EchoClient echoes the model name back as the completion.
+ assert_eq!(final_completion.ok_or("no ReturnToAgent")?, "direct/model");
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn run_returns_the_response_when_all_targets_have_clients(
+ ) -> Result<(), Box<dyn Error + Send + Sync>> {
+ // Every target has a client, so run serves every call via the
+ // default client and returns the trace + final response.
+ let (trace, response) = orch(target_set(&[("direct/model", true)]))
+ .run(Context::default(), request())
+ .await?;
+ // TestAlgo calls the first target; EchoClient echoes its name.
+ assert_eq!(
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ "direct/model"
+ );
+ assert_eq!(trace[0].selected_model(), "direct/model");
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn run_errors_when_a_target_lacks_a_client() -> Result<(), Box<dyn Error + Send + Sync>> {
+ // A client-less target has no default client to serve its offloaded call, so
+ // driving it to completion errors.
+ assert!(orch(target_set(&[("offload/model", false)]))
+ .run(Context::default(), request())
+ .await
+ .is_err());
+ Ok(())
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 12)]
+ async fn requests_are_processed_in_parallel() -> Result<(), Box<dyn Error + Send + Sync>> {
+ use std::time::Duration;
+ use tokio::sync::Barrier;
+
+ const N: usize = 12;
+
+ // A client that blocks until all N concurrent calls have arrived. If
+ // requests were serialized (one algorithm behind a `Mutex`), only one
+ // call could be in flight, the barrier would never reach N, and the test
+ // would time out. It passes only because the shared algorithm is driven
+ // concurrently across requests.
+ struct BarrierClient {
+ barrier: Arc<Barrier>,
+ }
+
+ #[async_trait]
+ impl RoutedLlmClient for BarrierClient {
+ async fn call(
+ &self,
+ _ctx: Context,
+ _request: Request,
+ decision: Arc<dyn Decision>,
+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {
+ self.barrier.wait().await;
+ Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(
+ None,
+ decision.selected_model().to_string(),
+ )),
+ metadata: None,
+ })
+ }
+ }
+
+ let barrier = Arc::new(Barrier::new(N));
+ let targets = LlmTargetSet::new(vec![LlmTarget {
+ semantic_name: "m".to_string(),
+ llm_client: Some(Arc::new(BarrierClient {
+ barrier: barrier.clone(),
+ })),
+ }]);
+ // One shared algorithm driven by many concurrent requests.
+ let algo = orch(targets);
+
+ let mut handles = Vec::new();
+ for _ in 0..N {
+ let algo = algo.clone();
+ handles.push(tokio::spawn(async move {
+ algo.run(Context::default(), request())
+ .await
+ .map(|(_, response)| {
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default()
+ })
+ }));
+ }
+
+ for handle in handles {
+ // The timeout turns a serialization deadlock into a failure, not a hang.
+ let completion = tokio::time::timeout(Duration::from_secs(5), handle).await???;
+ assert_eq!(completion, "m");
+ }
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn offload_error_propagates_back_to_the_algorithm(
+ ) -> Result<(), Box<dyn Error + Send + Sync>> {
+ // A client-less target offloads its call; we fulfill the promise with an
+ // Err, which must flow back through `call_llm_target` into the algorithm and
+ // out as an error step — not a response.
+ let stream =
+ orch(target_set(&[("offload/model", false)])).run_stream(Context::default(), request());
+ tokio::pin!(stream);
+
+ let mut saw_error = false;
+ while let Some(step) = stream.next().await {
+ match step {
+ Ok(Step::CallLlm(call)) => {
+ call.respond(Err("upstream model call failed".into()))?;
+ }
+ Ok(Step::Decision(_)) => {}
+ Ok(Step::ReturnToAgent(..)) => {
+ return Err("expected the offload error to propagate, got a response".into());
+ }
+ Err(err) => {
+ // The algorithm's `call_llm_target` saw the error via the promise.
+ assert!(err.to_string().contains("upstream model call failed"));
+ saw_error = true;
+ }
+ }
+ }
+
+ assert!(saw_error, "expected an error step");
+ Ok(())
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+ async fn run_caps_concurrent_calls() -> Result<(), Box<dyn Error + Send + Sync>> {
+ use std::sync::atomic::{AtomicUsize, Ordering};
+ use std::time::Duration;
+ use tokio::sync::{mpsc, Semaphore};
+
+ // Mirrors `run`'s private `MAX_CONCURRENT_CALLS`. The algorithm fans out more
+ // calls than the cap; only `CAP` should ever be in flight, the rest wait.
+ const CAP: usize = 10;
+ const TOTAL_CALLS: usize = CAP + 5;
+
+ // Records peak concurrency and holds each call open until the gate is released,
+ // so calls pile up and the cap can be observed.
+ struct ProbeClient {
+ current: Arc<AtomicUsize>,
+ max: Arc<AtomicUsize>,
+ entered: mpsc::UnboundedSender<()>,
+ gate: Arc<Semaphore>,
+ }
+
+ #[async_trait]
+ impl RoutedLlmClient for ProbeClient {
+ async fn call(
+ &self,
+ _ctx: Context,
+ _request: Request,
+ decision: Arc<dyn Decision>,
+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {
+ let now = self.current.fetch_add(1, Ordering::SeqCst) + 1;
+ self.max.fetch_max(now, Ordering::SeqCst);
+ let _ = self.entered.send(());
+ // Block until the test opens the gate.
+ let permit = self.gate.acquire().await.map_err(|e| e.to_string())?;
+ drop(permit);
+ self.current.fetch_sub(1, Ordering::SeqCst);
+ Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(
+ None,
+ decision.selected_model().to_string(),
+ )),
+ metadata: None,
+ })
+ }
+ }
+
+ // Fans out `n` concurrent calls to one target.
+ struct FanOut {
+ n: usize,
+ target: LlmTarget,
+ }
+
+ #[async_trait]
+ impl Algorithm for FanOut {
+ async fn create_run_task(
+ self: Arc<Self>,
+ ctx: Context,
+ driver: Driver,
+ _request: Request,
+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {
+ let calls = (0..self.n).map(|i| {
+ let driver = driver.clone();
+ let ctx = ctx.clone();
+ let target = self.target.clone();
+ async move {
+ let decision: Arc<dyn Decision> = Arc::new(TestDecision {
+ model: format!("m{i}"),
+ });
+ driver
+ .call_llm_target(ctx, &target, request(), decision)
+ .await
+ }
+ });
+ futures::future::join_all(calls).await;
+ Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(None, "done".to_string())),
+ metadata: None,
+ })
+ }
+ }
+
+ let current = Arc::new(AtomicUsize::new(0));
+ let max = Arc::new(AtomicUsize::new(0));
+ let (entered_tx, mut entered_rx) = mpsc::unbounded_channel();
+ let gate = Arc::new(Semaphore::new(0)); // starts closed
+
+ let client = Arc::new(ProbeClient {
+ current: current.clone(),
+ max: max.clone(),
+ entered: entered_tx,
+ gate: gate.clone(),
+ }) as Arc<dyn RoutedLlmClient>;
+ let target = LlmTarget {
+ semantic_name: "m".to_string(),
+ llm_client: Some(client),
+ };
+
+ let algo: Arc<dyn Algorithm> = Arc::new(FanOut {
+ n: TOTAL_CALLS,
+ target,
+ });
+ let handle = tokio::spawn(async move { algo.run(Context::default(), request()).await });
+
+ // Exactly `CAP` calls should enter; each `recv` times out into an error if the
+ // run dispatched fewer than the cap.
+ for _ in 0..CAP {
+ tokio::time::timeout(Duration::from_secs(5), entered_rx.recv())
+ .await
+ .map_err(|_| "timed out waiting for a call to enter")?;
+ }
+ assert_eq!(
+ current.load(Ordering::SeqCst),
+ CAP,
+ "exactly the cap should be in flight"
+ );
+ // No further call enters while the cap is saturated — extra steps wait for capacity.
+ assert!(
+ tokio::time::timeout(Duration::from_millis(200), entered_rx.recv())
+ .await
+ .is_err(),
+ "a call beyond the cap entered while {CAP} were already in flight"
+ );
+
+ // Release the gate; every call completes and the run finishes.
+ gate.add_permits(TOTAL_CALLS);
+ let (_trace, response) = tokio::time::timeout(Duration::from_secs(5), handle).await???;
+ assert_eq!(
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ "done"
+ );
+ assert_eq!(
+ max.load(Ordering::SeqCst),
+ CAP,
+ "peak in-flight should equal the cap"
+ );
+ Ok(())
+ }
+}
diff --git a/crates/libsy/src/driver.rs b/crates/libsy/src/core/driver.rs
similarity index 98%
rename from crates/libsy/src/driver.rs
rename to crates/libsy/src/core/driver.rs
index b0bb0c770..093b6e306 100644
--- a/crates/libsy/src/driver.rs
+++ b/crates/libsy/src/core/driver.rs
@@ -3,9 +3,10 @@
//! # driver — a type-erased promise-over-a-stream request pump
//!
-//! [`TypeErasedDriver`] is the generic offload primitive; the crate root builds the
-//! libsy-typed `Driver` on top of it. This module has no dependency on the rest of the
-//! crate — the coupling is one-directional (`lib.rs` → `driver.rs`).
+//! [`TypeErasedDriver`] is the generic offload primitive; its sibling
+//! [`algorithm`](super::algorithm) builds the libsy-typed `Driver` on top of it. This
+//! module has no dependency on the rest of the crate — the coupling is one-directional
+//! (`core/algorithm.rs` → `core/driver.rs`).
//!
//! A [`TypeErasedDriver`] lets a *producer* (e.g. a routing algorithm) fulfill
//! arbitrary requests by publishing promises onto a stream that a single *consumer*
diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs
index 2cf6ecf3b..2f8eebca8 100644
--- a/crates/libsy/src/lib.rs
+++ b/crates/libsy/src/lib.rs
@@ -54,947 +54,14 @@
//!
//! ## Algorithms
//!
-//! [`RandomAlgo`] provides uniform random routing in the core crate.
+//! Concrete algorithms live in [`algorithms`]:
//!
-//! [`LlmClassifierOrchAlgo`] classifies with one model, then routes to a
-//! strong/weak model depending on the classifier's choice.
-
-mod algorithms;
-pub use algorithms::llm_class::{ClassifierDecision, ClassifierTier, LlmClassifierOrchAlgo};
-pub use algorithms::noop::{NoopAlgo, NoopDecision};
-pub use algorithms::rand::{RandomAlgo, RandomDecision};
-
-mod driver;
-
-use std::{error::Error, pin::Pin, sync::Arc};
-
-use async_trait::async_trait;
-use futures::{Stream, StreamExt};
-
-/// The request/response protocol types, re-exported from [`switchyard_protocol`].
-/// [`LlmRequest`] is the normalized request; [`AggLlmResponse`] is the buffered response;
-/// [`LlmResponseChunk`] is one streaming event; [`LlmResponse`] is the streamed response
-/// (a live [`LlmResponseStream`] or the terminal aggregate).
-pub use switchyard_protocol::{
- AggLlmResponse, Context, Decision, LlmRequest, LlmResponse, LlmResponseChunk,
- LlmResponseStream, Metadata, Request, Response, RoutedLlmClient,
-};
-
-use crate::driver::{DriverRequest, DriverStep, TypeErasedDriver};
-
-/// Shorthand for the crate's boxed, thread-safe error type.
-type BoxErr = Box<dyn Error + Send + Sync>;
-
-/// A boxed, `Send` stream of [`Step`]s — the output of
-/// [`Algorithm::run_stream`]. Boxed so the trait method that produces it keeps
-/// `Arc<dyn Algorithm>` object-safe.
-pub type StepStream = Pin<Box<dyn Stream<Item = Result<Step, BoxErr>> + Send>>;
-
-/// Agentic-stack events fed to an algorithm out of band via
-/// [`Algorithm::process_signals`] (e.g. tool results, budget updates).
-///
-/// A placeholder today; a stateful algorithm can begin consuming signals as the
-/// enum grows without changing the orchestrator contract.
-#[derive(Clone)]
-pub struct Signals {}
-
-/// A request paired with the routing [`Decision`] that produced it — the offload
-/// payload a host reads (via [`CallLlmRequest::get_routed`]) to serve the call.
-///
-/// The two model identifiers live in separate, unambiguous places: the model to
-/// call is [`decision.selected_model()`](Decision::selected_model), while
-/// `request.llm_request.model` is the *inbound* name the agent asked for (libsy
-/// never overwrites it). A client maps `selected_model()` to the provider model
-/// id it hits.
-#[derive(Clone)]
-pub struct RoutedRequest {
- /// The request to serve; its `model` is the agent's original name.
- pub request: Request,
- /// The routing decision behind this call; `selected_model()` is the model to hit.
- pub decision: Arc<dyn Decision>,
- /// The client that serves this call by default, or `None` when the routed target
- /// had no client. Rides along on the offloaded call so a host driving the stream
- /// can serve it by default or override it with its own transport.
- pub default_client: Option<Arc<dyn RoutedLlmClient>>,
- /// The request's cross-cutting context, carried through the offload so whoever
- /// serves the call (libsy's own `run`, or a host driving the stream) hands it to
- /// [`RoutedLlmClient::call`].
- pub ctx: Context,
-}
-
-/// The host-facing half of an offloaded model call, surfaced inside [`Step::CallLlm`].
-///
-/// Wraps a [`DriverRequest`] whose payload is a [`RoutedRequest`]. The host reads the
-/// routed request ([`get_routed`](Self::get_routed)) and the decision behind it
-/// ([`get_decision`](Self::get_decision)), performs (or delegates) the model call, and
-/// fulfills it with [`respond`](Self::respond) — unblocking the algorithm's
-/// [`Driver::call_llm`] on the other side.
-pub struct CallLlmRequest {
- inner: DriverRequest,
- routed: RoutedRequest,
-}
-
-impl CallLlmRequest {
- /// Wrap a driver request whose payload is a [`RoutedRequest`]. Caches an owned copy
- /// so the accessors are plain field reads.
- fn new(inner: DriverRequest) -> Self {
- // The payload is always a `RoutedRequest` (set by `Driver::call_llm`); a
- // mismatch would be a libsy bug, not a runtime condition.
- let routed = match inner.request::<RoutedRequest>() {
- Ok(routed) => routed.clone(),
- Err(_) => unreachable!("CallLlmRequest payload is always a RoutedRequest"),
- };
- Self { inner, routed }
- }
-
- /// The routed request the host should serve. Its
- /// [`default_client`](RoutedRequest::default_client) serves the call by default,
- /// and its `decision.selected_model()` names the model to hit.
- pub fn get_routed(&self) -> &RoutedRequest {
- &self.routed
- }
-
- /// The model request to perform (the [`Request`] inside the routed request).
- pub fn get_request(&self) -> &Request {
- &self.get_routed().request
- }
-
- /// The decision that led to this call — its `selected_model()` is the model to hit.
- pub fn get_decision(&self) -> &dyn Decision {
- self.get_routed().decision.as_ref()
- }
-
- /// Fulfill the promise with the caller's model-call result. Pass `Err(..)` to
- /// propagate a failed model call back to the algorithm. Consumes the promise: it
- /// can only be fulfilled once.
- pub fn respond(self, result: Result<Response, BoxErr>) -> Result<(), BoxErr> {
- self.inner.respond::<Response>(result)
- }
-}
-
-/// The offload channel handed to an algorithm's
-/// [`create_run_task`](Algorithm::create_run_task). The algorithm makes model calls
-/// with [`call_llm_target`](Self::call_llm_target) (or [`call_llm`](Self::call_llm)) and
-/// publishes its [`Decision`]s with [`info`](Self::info); each call is offloaded to the
-/// request's [`Step`] stream and awaits the consumer's response. The step channel is
-/// bounded, so the consumer paces the algorithm one step at a time.
-#[derive(Clone)]
-pub struct Driver {
- driver: TypeErasedDriver,
-}
-
-impl Driver {
- /// Build an empty driver with its step channel ready. Created per call by
- /// [`run_stream`](Algorithm::run_stream).
- pub(crate) fn new() -> Self {
- Self {
- driver: TypeErasedDriver::new(),
- }
- }
-
- /// Offload a model call: publish `routed` as a [`Step::CallLlm`] and await the
- /// consumer's [`Response`]. The call's context travels inside
- /// [`routed.ctx`](RoutedRequest::ctx). Errors if the stream is closed or the call failed.
- pub async fn call_llm(&self, routed: RoutedRequest) -> Result<Response, BoxErr> {
- self.driver
- .fulfill_request::<RoutedRequest, Response>(routed.ctx.clone(), routed)
- .await
- }
-
- /// Offload a call to `target`: pair `request` with `decision` and the target's
- /// default client into a [`RoutedRequest`], then publish it (see
- /// [`call_llm`](Self::call_llm)). The convenience most algorithms use;
- /// `decision.selected_model()` names the model to hit, and `request`'s
- /// `model` is left untouched.
- pub async fn call_llm_target(
- &self,
- ctx: Context,
- target: &LlmTarget,
- request: Request,
- decision: Arc<dyn Decision>,
- ) -> Result<Response, BoxErr> {
- self.call_llm(RoutedRequest {
- request,
- decision,
- default_client: target.llm_client.clone(),
- ctx,
- })
- .await
- }
-
- /// Publish a routing [`Decision`] as a [`Step::Decision`] on the stream.
- pub async fn info(&self, ctx: Context, decision: Arc<dyn Decision>) -> Result<(), BoxErr> {
- self.driver.info(ctx, decision).await
- }
-
- /// Emit the terminal step: [`Step::ReturnToAgent`] on `Ok`, or an `Err` stream
- /// item on failure. Internal: called once by [`run_stream`](Algorithm::run_stream)
- /// when the algorithm finishes.
- pub(crate) async fn finish(
- &self,
- ctx: Context,
- result: Result<Response, BoxErr>,
- ) -> Result<(), BoxErr> {
- match result {
- Ok(response) => self.driver.done(ctx, response).await,
- Err(err) => self.driver.fail(ctx, err).await,
- }
- }
-
- /// Transform the raw driver stream into a stream of [`Step`]s. Internal: the
- /// consumer stream is taken (once) by [`run_stream`](Algorithm::run_stream). A
- /// payload that does not match the expected type for its step becomes an `Err` item.
- pub(crate) fn stream(&self) -> impl Stream<Item = Result<Step, BoxErr>> {
- self.driver.stream().map(|item| match item? {
- DriverStep::Request(req) => Ok(Step::CallLlm(Box::new(CallLlmRequest::new(req)))),
- DriverStep::Info(payload) => payload
- .downcast::<Arc<dyn Decision>>()
- .map(|decision| Step::Decision(*decision))
- .map_err(|_| "driver: info payload was not a Decision".into()),
- DriverStep::Done(payload) => payload
- .downcast::<Response>()
- .map(Step::ReturnToAgent)
- .map_err(|_| "driver: done payload was not a Response".into()),
- })
- }
-}
-
-impl Default for Driver {
- fn default() -> Self {
- Self::new()
- }
-}
-
-/// One item in the stream returned by [`Driver::stream`] / [`Algorithm::run_stream`].
-pub enum Step {
- /// The algorithm needs this model call performed. The host serves it (optionally
- /// via [`RoutedRequest::default_client`]) and fulfills it with
- /// [`CallLlmRequest::respond`]. Boxed: it is by far the largest variant.
- CallLlm(Box<CallLlmRequest>),
- /// A routing decision the algorithm made, published via [`Driver::info`] as it
- /// happens (rather than collected into a trace returned at the end).
- Decision(Arc<dyn Decision>),
- /// The algorithm finished with its final response — the last step of a run.
- ReturnToAgent(Box<Response>),
-}
-
-/// A named routing target: a `semantic_name` an algorithm routes by, and an optional
-/// [`RoutedLlmClient`] to serve its calls. An algorithm hands a target to
-/// [`Driver::call_llm_target`]; the client rides along as
-/// [`RoutedRequest::default_client`] for the stream consumer to serve or override.
-#[derive(Clone)]
-pub struct LlmTarget {
- /// The routing label an algorithm selects this target by — a logical tier like
- /// `"strong"`, or the model id when they coincide. Mapping it to a provider model
- /// id is the client's concern, never the algorithm's.
- pub semantic_name: String,
- /// The client that serves this target's calls by default, or `None` (then the
- /// stream consumer must serve them).
- pub llm_client: Option<Arc<dyn RoutedLlmClient>>,
-}
-
-/// The set of targets an algorithm may route among. An algorithm is constructed
-/// with one and picks targets by position ([`targets`](Self::targets)) or by name
-/// ([`get_target`](Self::get_target)).
-#[derive(Clone)]
-pub struct LlmTargetSet {
- targets: Vec<LlmTarget>,
-}
-
-impl LlmTargetSet {
- /// Build a target set from a list of targets.
- pub fn new(targets: Vec<LlmTarget>) -> Self {
- Self { targets }
- }
-
- /// All targets in the set — e.g. for an algorithm to select among.
- pub fn targets(&self) -> &[LlmTarget] {
- &self.targets
- }
-
- /// Look up a target by name; errors if no target has that name.
- pub fn get_target(&self, name: &str) -> Result<LlmTarget, Box<dyn Error + Send + Sync>> {
- self.targets
- .iter()
- .find(|t| t.semantic_name == name)
- .cloned()
- .ok_or(format!("Target {} not found", name).into())
- }
-}
-
-/// An optimization strategy. Implement [`create_run_task`](Self::create_run_task);
-/// callers drive it with the provided [`run`](Self::run) (serve calls, get the answer)
-/// or [`run_stream`](Self::run_stream) (drive the [`Step`] stream yourself).
-///
-/// Methods take `self: Arc<Self>`: one algorithm (`Arc<dyn Algorithm>`) is shared across
-/// requests and run concurrently, so it owns its thread-safety. Stateless algorithms
-/// (the reference routers) get this for free; a stateful one uses interior mutability
-/// over just its own state.
-#[async_trait]
-pub trait Algorithm: Send + Sync + 'static {
- /// Run one request to completion: make model calls with [`Driver::call_llm_target`],
- /// publish [`Decision`]s with [`Driver::info`], and return the final [`Response`].
- /// The method an algorithm implements; [`run`](Self::run) / [`run_stream`](Self::run_stream)
- /// drive it. `ctx` carries cross-cutting request state (empty today).
- async fn create_run_task(
- self: Arc<Self>,
- ctx: Context,
- driver: Driver,
- request: Request,
- ) -> Result<Response, Box<dyn Error + Send + Sync>>;
-
- /// Feed the algorithm agentic-stack events (tool results, budgets, etc.). The
- /// reference algorithms ignore signals; a stateful algorithm updates its own
- /// (interior-mutable) state. Takes `self: Arc<Self>` like the other run methods.
- #[allow(unused_variables)]
- async fn process_signals(
- self: Arc<Self>,
- signals: Signals,
- ) -> Result<(), Box<dyn Error + Send + Sync>> {
- Ok(())
- }
-
- /// Run one request as a stream of [`Step`]s (provided). The algorithm runs on its
- /// own task; drive the stream: serve each [`Step::CallLlm`] (via its
- /// [`default_client`](RoutedRequest::default_client) or your own transport) and read
- /// [`Step::Decision`]s until the final [`Step::ReturnToAgent`]. The step channel is
- /// bounded, so pulling paces the algorithm; each call is independent, so many run
- /// concurrently.
- fn run_stream(self: Arc<Self>, ctx: Context, request: Request) -> StepStream {
- // This call's own driver: take its consumer stream, hand a producer-side clone to
- // the algorithm task, and keep one to emit the terminal step. The task blocks
- // publishing a step until the consumer pulls the previous one.
- let driver = Driver::new();
- let stream = driver.stream();
- tokio::spawn(async move {
- let outcome = self
- .create_run_task(ctx.clone(), driver.clone(), request)
- .await;
- let _ = driver.finish(ctx, outcome).await;
- });
- Box::pin(stream)
- }
-
- /// Run one request to completion, serving each offloaded call with its
- /// [`RoutedRequest::default_client`], and return the decision trace plus the final
- /// [`Response`]. Provided: drives [`run_stream`](Self::run_stream) internally,
- /// collecting each [`Step::Decision`]. Use it when the algorithm holds its own model
- /// clients and the host wants the answer (and the decisions behind it); drive
- /// [`run_stream`](Self::run_stream) instead to serve the calls yourself. The returned
- /// [`Response`] is whatever the algorithm produced — a buffered [`LlmResponse::Agg`]
- /// or a live [`LlmResponse::Stream`]; the caller decides whether to
- /// [`aggregate`](LlmResponse::aggregate) it or forward its chunks. Errors if a routed
- /// target has no client to serve its call, or the algorithm fails.
- async fn run(
- self: Arc<Self>,
- ctx: Context,
- request: Request,
- ) -> Result<(Vec<Arc<dyn Decision>>, Response), Box<dyn Error + Send + Sync>> {
- // Serve up to this many offloaded calls concurrently, so an algorithm that
- // fans out (e.g. an ensemble) isn't serialized on the client.
- const MAX_CONCURRENT_CALLS: usize = 10;
-
- // Serve one offloaded call with its target's default client. A failed *model*
- // call is forwarded to the algorithm via `respond`; this errors only on an
- // infrastructure failure (no default client, or the promise was dropped).
- async fn serve(call: CallLlmRequest) -> Result<(), Box<dyn Error + Send + Sync>> {
- let routed = call.get_routed().clone();
- let client = routed.default_client.clone().ok_or_else(|| {
- format!(
- "run: target '{}' has no client to serve the call",
- routed.decision.selected_model()
- )
- })?;
- call.respond(
- client
- .call(routed.ctx, routed.request, routed.decision)
- .await,
- )
- }
-
- let stream = self.run_stream(ctx, request);
- tokio::pin!(stream);
-
- let mut trace: Vec<Arc<dyn Decision>> = Vec::new();
- let mut in_flight = futures::stream::FuturesUnordered::new();
- let mut final_response: Option<Response> = None;
- let mut stream_open = true;
-
- while stream_open || !in_flight.is_empty() {
- tokio::select! {
- // Surface a failed serve as soon as it completes.
- Some(result) = in_flight.next(), if !in_flight.is_empty() => result?,
- // Pull the next step, unless the stream ended or we're at the cap.
- step = stream.next(), if stream_open && in_flight.len() < MAX_CONCURRENT_CALLS => {
- match step {
- None => stream_open = false,
- Some(item) => match item? {
- Step::CallLlm(call) => in_flight.push(serve(*call)),
- Step::Decision(decision) => trace.push(decision),
- Step::ReturnToAgent(response) => {
- final_response = Some(*response);
- stream_open = false;
- }
- },
- }
- }
- }
- }
-
- // Return whatever the algorithm produced — buffered or streamed. The caller
- // decides whether to aggregate the response or forward its chunks.
- final_response
- .map(|response| (trace, response))
- .ok_or_else(|| "run: stream ended without a final response".into())
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use futures::StreamExt;
- use switchyard_protocol::{completion_text, text_request, text_response};
-
- /// Mock client that echoes back the target name it was called with.
- struct EchoClient;
-
- #[async_trait]
- impl RoutedLlmClient for EchoClient {
- async fn call(
- &self,
- _ctx: Context,
- _request: Request,
- decision: Arc<dyn Decision>,
- ) -> Result<Response, Box<dyn Error + Send + Sync>> {
- // Echo back the model the algorithm routed to (the decision's selection).
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(
- None,
- decision.selected_model().to_string(),
- )),
- metadata: None,
- })
- }
- }
-
- /// Trivial decision + algo used only to exercise the orchestrator: calls the
- /// first target and returns its response with a one-item trace.
- struct TestDecision {
- model: String,
- }
-
- impl Decision for TestDecision {
- fn selected_model(&self) -> &str {
- &self.model
- }
- fn reasoning(&self) -> Option<&str> {
- None
- }
- fn as_any(&self) -> &dyn std::any::Any {
- self
- }
- }
-
- struct TestAlgo {
- target_set: LlmTargetSet,
- }
-
- #[async_trait]
- impl Algorithm for TestAlgo {
- async fn create_run_task(
- self: Arc<Self>,
- ctx: Context,
- driver: Driver,
- request: Request,
- ) -> Result<Response, Box<dyn Error + Send + Sync>> {
- let target = self
- .target_set
- .targets()
- .first()
- .ok_or("no targets")?
- .clone();
- let decision: Arc<dyn Decision> = Arc::new(TestDecision {
- model: target.semantic_name.clone(),
- });
- driver.info(ctx.clone(), decision.clone()).await?;
- driver
- .call_llm_target(ctx, &target, request, decision)
- .await
- }
- }
-
- /// Build a shared `TestAlgo` over the given target set.
- fn orch(target_set: LlmTargetSet) -> Arc<dyn Algorithm> {
- Arc::new(TestAlgo { target_set })
- }
-
- fn request() -> Request {
- Request {
- llm_request: text_request(Some("auto".to_string()), "hi".to_string()),
- raw_request: None,
- metadata: None,
- }
- }
-
- /// `(name, has_client)` — `has_client: false` builds a target with no default client.
- fn target_set(names: &[(&str, bool)]) -> LlmTargetSet {
- let targets = names
- .iter()
- .map(|(name, has_client)| LlmTarget {
- semantic_name: name.to_string(),
- llm_client: has_client.then(|| Arc::new(EchoClient) as Arc<dyn RoutedLlmClient>),
- })
- .collect();
- LlmTargetSet::new(targets)
- }
-
- /// Client that serves a call as a token stream — its `call` returns
- /// [`LlmResponse::Stream`] replaying `chunks` in order (as `Ok` items).
- struct StreamingClient {
- chunks: Vec<LlmResponseChunk>,
- }
-
- #[async_trait]
- impl RoutedLlmClient for StreamingClient {
- async fn call(
- &self,
- _ctx: Context,
- _request: Request,
- _decision: Arc<dyn Decision>,
- ) -> Result<Response, Box<dyn Error + Send + Sync>> {
- let stream = futures::stream::iter(self.chunks.clone().into_iter().map(Ok)).boxed();
- Ok(Response {
- llm_response: LlmResponse::Stream(stream),
- metadata: None,
- })
- }
- }
-
- /// Build a single-target algo whose one target streams `chunks`.
- fn streaming_orch(chunks: Vec<LlmResponseChunk>) -> Arc<dyn Algorithm> {
- let target = LlmTarget {
- semantic_name: "stream/model".to_string(),
- llm_client: Some(Arc::new(StreamingClient { chunks }) as Arc<dyn RoutedLlmClient>),
- };
- orch(LlmTargetSet::new(vec![target]))
- }
-
- #[tokio::test]
- async fn run_returns_a_streamed_response_the_caller_aggregates(
- ) -> Result<(), Box<dyn Error + Send + Sync>> {
- // A streaming client -> its chunks flow through the promise and `ReturnToAgent`,
- // and `run` returns the live stream untouched for the caller to fold.
- let orch = streaming_orch(vec![
- LlmResponseChunk::MessageStart {
- id: Some("m1".to_string()),
- model: Some("stream/model".to_string()),
- },
- LlmResponseChunk::TextDelta {
- index: 0,
- text: "hel".to_string(),
- },
- LlmResponseChunk::TextDelta {
- index: 0,
- text: "lo".to_string(),
- },
- LlmResponseChunk::MessageStop {
- reason: Some("stop".to_string()),
- },
- ]);
- let (trace, response) = orch.run(Context::default(), request()).await?;
- // `run` handed back the live stream; the caller folds it to a buffered aggregate.
- let agg = response.llm_response.into_agg().await?;
- assert_eq!(completion_text(&agg), "hello");
- assert_eq!(agg.model.as_deref(), Some("stream/model"));
- assert_eq!(trace.len(), 1);
- Ok(())
- }
-
- #[tokio::test]
- async fn aggregating_a_streamed_response_propagates_a_mid_stream_error(
- ) -> Result<(), Box<dyn Error + Send + Sync>> {
- // `run` succeeds and returns the stream; the in-band `Error` chunk surfaces only
- // when the caller aggregates it.
- let orch = streaming_orch(vec![
- LlmResponseChunk::TextDelta {
- index: 0,
- text: "partial".to_string(),
- },
- LlmResponseChunk::Error {
- message: "upstream exploded".to_string(),
- },
- ]);
- let (_, response) = orch.run(Context::default(), request()).await?;
- match response.llm_response.into_agg().await {
- Ok(_) => Err("expected a mid-stream error, got an aggregate".into()),
- Err(err) => {
- assert!(err.to_string().contains("upstream exploded"));
- Ok(())
- }
- }
- }
-
- #[tokio::test]
- async fn run_offloads_via_promise_then_returns_to_agent(
- ) -> Result<(), Box<dyn Error + Send + Sync>> {
- // A client-less target -> its call is offloaded via a promise the
- // orchestrator surfaces as a `CallLlm` step for us to fulfill.
- let stream =
- orch(target_set(&[("offload/model", false)])).run_stream(Context::default(), request());
- tokio::pin!(stream);
-
- let mut saw_call = false;
- let mut final_completion = None;
- while let Some(step) = stream.next().await {
- match step? {
- Step::CallLlm(call) => {
- saw_call = true;
- // The decision rode along with the promise.
- assert_eq!(call.get_decision().selected_model(), "offload/model");
- // Fulfilling the promise is the "real" model call the caller makes.
- call.respond(Ok(Response {
- llm_response: LlmResponse::Agg(text_response(
- None,
- "fulfilled".to_string(),
- )),
- metadata: None,
- }))?;
- }
- Step::Decision(decision) => {
- assert_eq!(decision.selected_model(), "offload/model");
- }
- Step::ReturnToAgent(response) => {
- final_completion = Some(
- response
- .llm_response
- .as_agg()
- .map(completion_text)
- .unwrap_or_default(),
- );
- }
- }
- }
-
- assert!(saw_call, "expected a CallLlm step before ReturnToAgent");
- assert_eq!(
- final_completion.ok_or("no ReturnToAgent step")?,
- "fulfilled"
- );
- Ok(())
- }
-
- #[tokio::test]
- async fn client_backed_target_offloads_with_a_default_client(
- ) -> Result<(), Box<dyn Error + Send + Sync>> {
- // Every call now offloads to the stream; a client-backed target rides its
- // client along as `default_client` so the consumer can serve it by default.
- let stream =
- orch(target_set(&[("direct/model", true)])).run_stream(Context::default(), request());
- tokio::pin!(stream);
-
- let mut final_completion = None;
- while let Some(step) = stream.next().await {
- match step? {
- Step::CallLlm(call) => {
- let routed = call.get_routed().clone();
- let client = routed
- .default_client
- .clone()
- .ok_or("expected a default client")?;
- let result = client
- .call(routed.ctx, routed.request, routed.decision)
- .await;
- call.respond(result)?;
- }
- Step::Decision(_) => {}
- Step::ReturnToAgent(response) => {
- final_completion = Some(
- response
- .llm_response
- .as_agg()
- .map(completion_text)
- .unwrap_or_default(),
- );
- }
- }
- }
-
- // EchoClient echoes the model name back as the completion.
- assert_eq!(final_completion.ok_or("no ReturnToAgent")?, "direct/model");
- Ok(())
- }
-
- #[tokio::test]
- async fn run_returns_the_response_when_all_targets_have_clients(
- ) -> Result<(), Box<dyn Error + Send + Sync>> {
- // Every target has a client, so run serves every call via the
- // default client and returns the trace + final response.
- let (trace, response) = orch(target_set(&[("direct/model", true)]))
- .run(Context::default(), request())
- .await?;
- // TestAlgo calls the first target; EchoClient echoes its name.
- assert_eq!(
- response
- .llm_response
- .as_agg()
- .map(completion_text)
- .unwrap_or_default(),
- "direct/model"
- );
- assert_eq!(trace[0].selected_model(), "direct/model");
- Ok(())
- }
-
- #[tokio::test]
- async fn run_errors_when_a_target_lacks_a_client() -> Result<(), Box<dyn Error + Send + Sync>> {
- // A client-less target has no default client to serve its offloaded call, so
- // driving it to completion errors.
- assert!(orch(target_set(&[("offload/model", false)]))
- .run(Context::default(), request())
- .await
- .is_err());
- Ok(())
- }
-
- #[tokio::test(flavor = "multi_thread", worker_threads = 12)]
- async fn requests_are_processed_in_parallel() -> Result<(), Box<dyn Error + Send + Sync>> {
- use std::time::Duration;
- use tokio::sync::Barrier;
-
- const N: usize = 12;
-
- // A client that blocks until all N concurrent calls have arrived. If
- // requests were serialized (one algorithm behind a `Mutex`), only one
- // call could be in flight, the barrier would never reach N, and the test
- // would time out. It passes only because the shared algorithm is driven
- // concurrently across requests.
- struct BarrierClient {
- barrier: Arc<Barrier>,
- }
-
- #[async_trait]
- impl RoutedLlmClient for BarrierClient {
- async fn call(
- &self,
- _ctx: Context,
- _request: Request,
- decision: Arc<dyn Decision>,
- ) -> Result<Response, Box<dyn Error + Send + Sync>> {
- self.barrier.wait().await;
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(
- None,
- decision.selected_model().to_string(),
- )),
- metadata: None,
- })
- }
- }
-
- let barrier = Arc::new(Barrier::new(N));
- let targets = LlmTargetSet::new(vec![LlmTarget {
- semantic_name: "m".to_string(),
- llm_client: Some(Arc::new(BarrierClient {
- barrier: barrier.clone(),
- })),
- }]);
- // One shared algorithm driven by many concurrent requests.
- let algo = orch(targets);
-
- let mut handles = Vec::new();
- for _ in 0..N {
- let algo = algo.clone();
- handles.push(tokio::spawn(async move {
- algo.run(Context::default(), request())
- .await
- .map(|(_, response)| {
- response
- .llm_response
- .as_agg()
- .map(completion_text)
- .unwrap_or_default()
- })
- }));
- }
-
- for handle in handles {
- // The timeout turns a serialization deadlock into a failure, not a hang.
- let completion = tokio::time::timeout(Duration::from_secs(5), handle).await???;
- assert_eq!(completion, "m");
- }
- Ok(())
- }
-
- #[tokio::test]
- async fn offload_error_propagates_back_to_the_algorithm(
- ) -> Result<(), Box<dyn Error + Send + Sync>> {
- // A client-less target offloads its call; we fulfill the promise with an
- // Err, which must flow back through `call_llm_target` into the algorithm and
- // out as an error step — not a response.
- let stream =
- orch(target_set(&[("offload/model", false)])).run_stream(Context::default(), request());
- tokio::pin!(stream);
-
- let mut saw_error = false;
- while let Some(step) = stream.next().await {
- match step {
- Ok(Step::CallLlm(call)) => {
- call.respond(Err("upstream model call failed".into()))?;
- }
- Ok(Step::Decision(_)) => {}
- Ok(Step::ReturnToAgent(..)) => {
- return Err("expected the offload error to propagate, got a response".into());
- }
- Err(err) => {
- // The algorithm's `call_llm_target` saw the error via the promise.
- assert!(err.to_string().contains("upstream model call failed"));
- saw_error = true;
- }
- }
- }
-
- assert!(saw_error, "expected an error step");
- Ok(())
- }
-
- #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
- async fn run_caps_concurrent_calls() -> Result<(), Box<dyn Error + Send + Sync>> {
- use std::sync::atomic::{AtomicUsize, Ordering};
- use std::time::Duration;
- use tokio::sync::{mpsc, Semaphore};
-
- // Mirrors `run`'s private `MAX_CONCURRENT_CALLS`. The algorithm fans out more
- // calls than the cap; only `CAP` should ever be in flight, the rest wait.
- const CAP: usize = 10;
- const TOTAL_CALLS: usize = CAP + 5;
-
- // Records peak concurrency and holds each call open until the gate is released,
- // so calls pile up and the cap can be observed.
- struct ProbeClient {
- current: Arc<AtomicUsize>,
- max: Arc<AtomicUsize>,
- entered: mpsc::UnboundedSender<()>,
- gate: Arc<Semaphore>,
- }
-
- #[async_trait]
- impl RoutedLlmClient for ProbeClient {
- async fn call(
- &self,
- _ctx: Context,
- _request: Request,
- decision: Arc<dyn Decision>,
- ) -> Result<Response, Box<dyn Error + Send + Sync>> {
- let now = self.current.fetch_add(1, Ordering::SeqCst) + 1;
- self.max.fetch_max(now, Ordering::SeqCst);
- let _ = self.entered.send(());
- // Block until the test opens the gate.
- let permit = self.gate.acquire().await.map_err(|e| e.to_string())?;
- drop(permit);
- self.current.fetch_sub(1, Ordering::SeqCst);
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(
- None,
- decision.selected_model().to_string(),
- )),
- metadata: None,
- })
- }
- }
-
- // Fans out `n` concurrent calls to one target.
- struct FanOut {
- n: usize,
- target: LlmTarget,
- }
-
- #[async_trait]
- impl Algorithm for FanOut {
- async fn create_run_task(
- self: Arc<Self>,
- ctx: Context,
- driver: Driver,
- _request: Request,
- ) -> Result<Response, Box<dyn Error + Send + Sync>> {
- let calls = (0..self.n).map(|i| {
- let driver = driver.clone();
- let ctx = ctx.clone();
- let target = self.target.clone();
- async move {
- let decision: Arc<dyn Decision> = Arc::new(TestDecision {
- model: format!("m{i}"),
- });
- driver
- .call_llm_target(ctx, &target, request(), decision)
- .await
- }
- });
- futures::future::join_all(calls).await;
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(None, "done".to_string())),
- metadata: None,
- })
- }
- }
-
- let current = Arc::new(AtomicUsize::new(0));
- let max = Arc::new(AtomicUsize::new(0));
- let (entered_tx, mut entered_rx) = mpsc::unbounded_channel();
- let gate = Arc::new(Semaphore::new(0)); // starts closed
-
- let client = Arc::new(ProbeClient {
- current: current.clone(),
- max: max.clone(),
- entered: entered_tx,
- gate: gate.clone(),
- }) as Arc<dyn RoutedLlmClient>;
- let target = LlmTarget {
- semantic_name: "m".to_string(),
- llm_client: Some(client),
- };
-
- let algo: Arc<dyn Algorithm> = Arc::new(FanOut {
- n: TOTAL_CALLS,
- target,
- });
- let handle = tokio::spawn(async move { algo.run(Context::default(), request()).await });
+//! [`algorithms::Random`](crate::algorithms::Random) provides uniform random routing.
+//!
+//! [`algorithms::LlmClassifier`](crate::algorithms::LlmClassifier) classifies
+//! with one model, then routes to a strong/weak model depending on the classifier's choice.
- // Exactly `CAP` calls should enter; each `recv` times out into an error if the
- // run dispatched fewer than the cap.
- for _ in 0..CAP {
- tokio::time::timeout(Duration::from_secs(5), entered_rx.recv())
- .await
- .map_err(|_| "timed out waiting for a call to enter")?;
- }
- assert_eq!(
- current.load(Ordering::SeqCst),
- CAP,
- "exactly the cap should be in flight"
- );
- // No further call enters while the cap is saturated — extra steps wait for capacity.
- assert!(
- tokio::time::timeout(Duration::from_millis(200), entered_rx.recv())
- .await
- .is_err(),
- "a call beyond the cap entered while {CAP} were already in flight"
- );
+mod core;
+pub use core::*;
- // Release the gate; every call completes and the run finishes.
- gate.add_permits(TOTAL_CALLS);
- let (_trace, response) = tokio::time::timeout(Duration::from_secs(5), handle).await???;
- assert_eq!(
- response
- .llm_response
- .as_agg()
- .map(completion_text)
- .unwrap_or_default(),
- "done"
- );
- assert_eq!(
- max.load(Ordering::SeqCst),
- CAP,
- "peak in-flight should equal the cap"
- );
- Ok(())
- }
-}
+pub mod algorithms;
diff --git a/crates/libsy-protocol/Cargo.toml b/crates/protocol/Cargo.toml
similarity index 100%
rename from crates/libsy-protocol/Cargo.toml
rename to crates/protocol/Cargo.toml
diff --git a/crates/libsy-protocol/LICENSE b/crates/protocol/LICENSE
similarity index 100%
rename from crates/libsy-protocol/LICENSE
rename to crates/protocol/LICENSE
diff --git a/crates/libsy-protocol/NOTICE b/crates/protocol/NOTICE
similarity index 100%
rename from crates/libsy-protocol/NOTICE
rename to crates/protocol/NOTICE
diff --git a/crates/libsy-protocol/README.md b/crates/protocol/README.md
similarity index 100%
rename from crates/libsy-protocol/README.md
rename to crates/protocol/README.md
diff --git a/crates/libsy-protocol/src/client.rs b/crates/protocol/src/client.rs
similarity index 100%
rename from crates/libsy-protocol/src/client.rs
rename to crates/protocol/src/client.rs
diff --git a/crates/libsy-protocol/src/envelope.rs b/crates/protocol/src/envelope.rs
similarity index 90%
rename from crates/libsy-protocol/src/envelope.rs
rename to crates/protocol/src/envelope.rs
index 9906bd866..1781d3637 100644
--- a/crates/libsy-protocol/src/envelope.rs
+++ b/crates/protocol/src/envelope.rs
@@ -15,6 +15,14 @@ pub struct Context {
pub values: HashMap<String, String>,
}
+/// Agentic-stack events fed to an algorithm out of band (e.g. tool results, budget
+/// updates) — in libsy, via `Algorithm::process_signals`.
+///
+/// A placeholder today; a stateful algorithm can begin consuming signals as the enum
+/// grows without changing the orchestrator contract.
+#[derive(Clone)]
+pub struct Signals {}
+
/// Correlation and routing metadata attached to a request or response.
///
/// All fields are optional; algorithms and observers use whichever are present
diff --git a/crates/libsy-protocol/src/format.rs b/crates/protocol/src/format.rs
similarity index 100%
rename from crates/libsy-protocol/src/format.rs
rename to crates/protocol/src/format.rs
diff --git a/crates/libsy-protocol/src/lib.rs b/crates/protocol/src/lib.rs
similarity index 100%
rename from crates/libsy-protocol/src/lib.rs
rename to crates/protocol/src/lib.rs
diff --git a/crates/libsy-protocol/src/llm.rs b/crates/protocol/src/llm.rs
similarity index 100%
rename from crates/libsy-protocol/src/llm.rs
rename to crates/protocol/src/llm.rs
diff --git a/crates/libsy-protocol/src/stream.rs b/crates/protocol/src/stream.rs
similarity index 100%
rename from crates/libsy-protocol/src/stream.rs
rename to crates/protocol/src/stream.rs
diff --git a/crates/switchyard-py/Cargo.toml b/crates/switchyard-py/Cargo.toml
index 5659430c1..074625a67 100644
--- a/crates/switchyard-py/Cargo.toml
+++ b/crates/switchyard-py/Cargo.toml
@@ -18,7 +18,7 @@ crate-type = ["cdylib", "rlib"]
[dependencies]
async-trait = "0.1"
futures-util = "0.3"
-libsy = { path = "../libsy" }
+switchyard-libsy = { path = "../libsy" }
pyo3 = { version = "0.28.3", features = ["abi3-py312", "extension-module"] }
pyo3-async-runtimes = { version = "0.28", features = ["tokio-runtime"] }
pythonize = "0.28.0"
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index a7e62ef32..02ce29037 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -7,13 +7,14 @@ use std::error::Error;
use std::sync::Arc;
use async_trait::async_trait;
-use libsy::{
- AggLlmResponse, Algorithm, Context, Decision, LlmResponse, LlmTarget, LlmTargetSet, NoopAlgo,
- RandomAlgo, Request, Response, RoutedLlmClient,
-};
use pyo3::exceptions::{PyTypeError, PyValueError};
use pyo3::prelude::*;
use serde_json::{json, Value};
+use switchyard_libsy::algorithms::{Noop, Random};
+use switchyard_libsy::{
+ AggLlmResponse, Algorithm, Context, Decision, LlmResponse, LlmTarget, LlmTargetSet, Request,
+ Response, RoutedLlmClient,
+};
use crate::errors::py_libsy_error;
use crate::py_serde::{from_python, to_python};
@@ -148,7 +149,7 @@ impl PyAlgorithm {
/// Construct the no-op reference algorithm.
#[pyfunction(name = "noop")]
fn noop_algorithm() -> PyAlgorithm {
- PyAlgorithm::new(Arc::new(NoopAlgo {}))
+ PyAlgorithm::new(Arc::new(Noop {}))
}
/// Construct uniform random routing over targets with Python clients.
@@ -161,9 +162,9 @@ fn random_algorithm(py: Python<'_>, targets: Vec<Py<PyLlmTarget>>) -> PyResult<P
.iter()
.map(|target| Ok(target.bind(py).try_borrow()?.clone_core(py)))
.collect::<PyResult<Vec<_>>>()?;
- Ok(PyAlgorithm::new(Arc::new(RandomAlgo::new(
- LlmTargetSet::new(targets),
- ))))
+ Ok(PyAlgorithm::new(Arc::new(Random::new(LlmTargetSet::new(
+ targets,
+ )))))
}
fn boxed_python_error(error: PyErr) -> BoxError {
diff --git a/crates/switchyard-translation/Cargo.toml b/crates/switchyard-translation/Cargo.toml
index 9e55e7e9c..7f15b75da 100644
--- a/crates/switchyard-translation/Cargo.toml
+++ b/crates/switchyard-translation/Cargo.toml
@@ -19,7 +19,7 @@ publish = ["crates-io"]
[dependencies]
serde = { version = "1", features = ["derive"] }
serde_json = "1"
-switchyard-protocol = { path = "../libsy-protocol", version = "0.1.0" }
+switchyard-protocol = { path = "../protocol", version = "0.1.0" }
thiserror = "2"
futures = "0.3"
async-stream = "0.3"