Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(llm-client): prepare routed completion candidates (#463)

Alex Fournier3ad68556e6a1
diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs
index 059675bbe..adce273b8 100644
--- a/crates/libsy-llm-client/src/lib.rs
+++ b/crates/libsy-llm-client/src/lib.rs
@@ -30,7 +30,7 @@ pub use client::{ModelConfig, TranslatingLlmClient};
pub use error::{LlmClientError, Result};
pub use observation::{LlmCallObservation, RunObservation, RunObserver};
pub use raw::RawResponse;
-pub use run::{ClientRouter, run};
+pub use run::{ClientRouter, decide, run};
pub use switchyard_translation::RawEventStream;
/// Registers process-wide compatibility gauges with the global meter provider.
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index df14f22f8..454ae5f30 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -8,10 +8,10 @@
//! consumer — it drives the stream with [`switchyard_libsy::drive`], hands routing-time calls to
//! a [`RoutedLlmClient`], and serves the terminal routing outcome.
//!
-//! libsy owns the stream mechanics; what this module adds is ordered candidate fallback and the
-//! `libsy.client_call` span around each candidate. Each candidate exhausts its backend retry
-//! budget before fallback advances, so the worst case is `candidates × (max_retries + 1)`
-//! upstream attempts plus every candidate's backoff.
+//! libsy owns the stream mechanics; what this module adds is per-target request preparation,
+//! ordered candidate fallback, and the `libsy.client_call` span around each candidate. Each
+//! candidate exhausts its backend retry budget before fallback advances, so the worst case is
+//! `candidates × (max_retries + 1)` upstream attempts plus every candidate's backoff.
use std::collections::HashMap;
use std::sync::Arc;
@@ -19,10 +19,11 @@ use std::time::Instant;
use http::StatusCode;
use parking_lot::Mutex;
-use switchyard_libsy::{Algorithm, CallModel, LibsyError, Result, drive};
+use switchyard_libsy::{Algorithm, CallModel, LibsyError, Result, RoutingOutcome, drive};
use switchyard_protocol::{
LlmClientError, ModelId, Request, Response, RoutedLlmClient, RoutingFallbackReason,
};
+use switchyard_translation::prepare_request_for_target;
use crate::observation::{LlmCallObservation, RunObservation, RunObserver};
use crate::{metrics, observability};
@@ -84,6 +85,7 @@ pub async fn run(
&algorithm_name,
&outcome.request,
&models,
+ CallPhase::Completion,
&observe,
)
.await;
@@ -103,6 +105,25 @@ pub async fn run(
result.map(|response| (selected_model_id, response))
}
+/// Run an algorithm to a routing decision without serving its terminal completion.
+///
+/// Routing-time calls are served normally. The returned request is prepared only for the selected
+/// target; use [`run`] to execute selected and fallback candidates.
+pub async fn decide(
+ algorithm: Arc<dyn Algorithm>,
+ clients: ClientRouter,
+ request: Request,
+) -> Result<RoutingOutcome> {
+ let routing_clients = clients.clone();
+ let mut outcome = drive(algorithm, request, move |call| {
+ serve(routing_clients.clone(), call, None)
+ })
+ .await?;
+ outcome.request =
+ clients.prepare_completion_request(outcome.request, &outcome.selected_model_id);
+ Ok(outcome)
+}
+
/// Emits completed routing calls after the outcome reveals whether one response became the answer.
fn emit_routing_observations(
observer: &Option<RunObserver>,
@@ -144,22 +165,32 @@ async fn serve(
&call.algorithm,
&call.request,
&call.models,
+ CallPhase::Routing,
&observe,
)
.await;
call.respond(result)
}
+enum CallPhase {
+ Routing,
+ Completion,
+}
+
/// Try candidates in order until one succeeds or a failure stops fallback.
async fn call_first_available(
clients: &ClientRouter,
algorithm: &str,
request: &Request,
models: &[ModelId],
+ phase: CallPhase,
observe: &(dyn Fn(LlmCallObservation) + Send + Sync),
) -> Result<Response> {
for (index, target) in models.iter().enumerate() {
- let request = request_for(request, target);
+ let request = match phase {
+ CallPhase::Routing => clients.prepare_routing_request(request.clone(), target),
+ CallPhase::Completion => clients.prepare_completion_request(request.clone(), target),
+ };
match call_one(
clients,
target,
@@ -298,13 +329,6 @@ fn fallback_reason(error: &LibsyError) -> Option<RoutingFallbackReason> {
}
}
-/// Clone a request and stamp the candidate model that should receive it.
-fn request_for(request: &Request, target: &ModelId) -> Request {
- let mut request = request.clone();
- request.llm_request.model = Some(target.to_string());
- request
-}
-
/// Resolves a routed call's selected model to the client that serves it.
///
/// An algorithm routes among named targets; which provider each target lives on is the
@@ -315,7 +339,13 @@ fn request_for(request: &Request, target: &ModelId) -> Request {
/// Cloning is cheap — the mapping is shared, so one router can serve every request.
#[derive(Clone)]
pub struct ClientRouter {
- routing: Arc<Routing>,
+ inner: Arc<ClientRouting>,
+}
+
+struct ClientRouting {
+ routing: Routing,
+ target_prompts: HashMap<ModelId, String>,
+ routing_answer_target: Option<ModelId>,
}
enum Routing {
@@ -328,8 +358,25 @@ enum Routing {
impl ClientRouter {
/// Build a router over `model name -> client`, for targets spread across providers.
pub fn new(by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>) -> Self {
+ Self::new_with_target_prompts(by_model, HashMap::new(), None)
+ }
+
+ /// Build a router with target prompts used by [`run`] and [`decide`].
+ ///
+ /// `routing_answer_target` identifies a routing-time call whose response may become the
+ /// answer. Pass `None` when routing only selects a later completion target. Prompt keys and
+ /// `routing_answer_target` use the resolved model IDs stored in `by_model`.
+ pub fn new_with_target_prompts(
+ by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>,
+ target_prompts: HashMap<ModelId, String>,
+ routing_answer_target: Option<ModelId>,
+ ) -> Self {
Self {
- routing: Arc::new(Routing::ByModel(by_model)),
+ inner: Arc::new(ClientRouting {
+ routing: Routing::ByModel(by_model),
+ target_prompts,
+ routing_answer_target,
+ }),
}
}
@@ -340,7 +387,11 @@ impl ClientRouter {
/// only duplicate that.
pub fn single(client: Arc<dyn RoutedLlmClient>) -> Self {
Self {
- routing: Arc::new(Routing::Single(client)),
+ inner: Arc::new(ClientRouting {
+ routing: Routing::Single(client),
+ target_prompts: HashMap::new(),
+ routing_answer_target: None,
+ }),
}
}
@@ -352,7 +403,7 @@ impl ClientRouter {
&self,
model: &ModelId,
) -> std::result::Result<&Arc<dyn RoutedLlmClient>, LlmClientError> {
- match self.routing.as_ref() {
+ match &self.inner.routing {
Routing::Single(client) => Ok(client),
Routing::ByModel(by_model) => {
by_model
@@ -363,6 +414,24 @@ impl ClientRouter {
}
}
}
+
+ /// Prepare a completion candidate with its configured target prompt.
+ fn prepare_completion_request(&self, mut request: Request, target: &ModelId) -> Request {
+ let prompt = self.inner.target_prompts.get(target).map(String::as_str);
+ prepare_request_for_target(&mut request.llm_request, target, prompt);
+ request
+ }
+
+ /// Prepare a routing call, adding a target prompt only when it generates a candidate answer.
+ fn prepare_routing_request(&self, mut request: Request, target: &ModelId) -> Request {
+ let prompt = if self.inner.routing_answer_target.as_ref() == Some(target) {
+ self.inner.target_prompts.get(target).map(String::as_str)
+ } else {
+ None
+ };
+ prepare_request_for_target(&mut request.llm_request, target, prompt);
+ request
+ }
}
impl FromIterator<(ModelId, Arc<dyn RoutedLlmClient>)> for ClientRouter {
@@ -381,8 +450,8 @@ mod tests {
use http::StatusCode;
use switchyard_libsy::{Driver, RoutingOutcome};
use switchyard_protocol::{
- LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, completion_text, text_request,
- text_response,
+ ContentBlock, LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, completion_text,
+ text_request, text_response,
};
use wiremock::matchers::method;
use wiremock::{Mock, MockServer, ResponseTemplate};
@@ -449,6 +518,7 @@ mod tests {
struct CandidateClient {
calls: Mutex<Vec<ModelId>>,
+ requests: Mutex<Vec<Request>>,
first: FirstOutcome,
}
@@ -457,6 +527,7 @@ mod tests {
async fn call(&self, request: Request) -> std::result::Result<Response, LlmClientError> {
let model = request.model_id().unwrap_or_default();
self.calls.lock().push(model.clone());
+ self.requests.lock().push(request);
if model == "weak" {
return match self.first {
FirstOutcome::ContextWindow => Err(LlmClientError::ContextWindowExceeded {
@@ -516,11 +587,25 @@ mod tests {
}
}
+ fn instruction_text(request: &Request) -> Vec<&str> {
+ request
+ .llm_request
+ .instructions
+ .iter()
+ .flat_map(|instruction| &instruction.content)
+ .filter_map(|block| match block {
+ ContentBlock::Text { text } => Some(text.as_str()),
+ _ => None,
+ })
+ .collect()
+ }
+
async fn run_candidates(
first: FirstOutcome,
) -> (Arc<CandidateClient>, Result<(ModelId, Response)>) {
let client = Arc::new(CandidateClient {
calls: Mutex::new(Vec::new()),
+ requests: Mutex::new(Vec::new()),
first,
});
let algorithm = Arc::new(CandidateAlgorithm {
@@ -540,6 +625,7 @@ mod tests {
async fn answered_outcome_does_not_make_a_second_model_call() -> Result<()> {
let client = Arc::new(CandidateClient {
calls: Mutex::new(Vec::new()),
+ requests: Mutex::new(Vec::new()),
first: FirstOutcome::StreamSuccess,
});
let observations = Arc::new(Mutex::new(Vec::new()));
@@ -569,6 +655,106 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn each_fallback_candidate_receives_only_its_own_prompt() -> Result<()> {
+ let client = Arc::new(CandidateClient {
+ calls: Mutex::new(Vec::new()),
+ requests: Mutex::new(Vec::new()),
+ first: FirstOutcome::ContextWindow,
+ });
+ let routed_client: Arc<dyn RoutedLlmClient> = client.clone();
+ let clients = ClientRouter::new_with_target_prompts(
+ HashMap::from([
+ (ModelId::from("weak"), Arc::clone(&routed_client)),
+ (ModelId::from("strong"), routed_client),
+ ]),
+ HashMap::from([
+ ("weak".into(), "weak prompt".to_string()),
+ ("strong".into(), "strong prompt".to_string()),
+ ]),
+ None,
+ );
+
+ run(
+ Arc::new(CandidateAlgorithm {
+ models: vec!["weak".into(), "strong".into()],
+ }),
+ clients,
+ request(),
+ None,
+ )
+ .await?;
+
+ let calls = client.requests.lock();
+ assert_eq!(calls.len(), 2);
+ assert_eq!(instruction_text(&calls[0]), ["weak prompt"]);
+ assert_eq!(instruction_text(&calls[1]), ["strong prompt"]);
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn decision_prompts_a_routing_response_target() -> Result<()> {
+ let client = Arc::new(CandidateClient {
+ calls: Mutex::new(Vec::new()),
+ requests: Mutex::new(Vec::new()),
+ first: FirstOutcome::StreamSuccess,
+ });
+ let routed_client: Arc<dyn RoutedLlmClient> = client.clone();
+ let clients = ClientRouter::new_with_target_prompts(
+ HashMap::from([(ModelId::from("answer"), routed_client)]),
+ HashMap::from([("answer".into(), "answer prompt".to_string())]),
+ Some("answer".into()),
+ );
+
+ decide(
+ Arc::new(AnsweredAlgorithm {
+ model: "answer".into(),
+ }),
+ clients,
+ request(),
+ )
+ .await?;
+
+ let calls = client.requests.lock();
+ assert_eq!(calls.len(), 1);
+ assert_eq!(instruction_text(&calls[0]), ["answer prompt"]);
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn decision_prepares_only_the_selected_request() -> Result<()> {
+ let client = Arc::new(CandidateClient {
+ calls: Mutex::new(Vec::new()),
+ requests: Mutex::new(Vec::new()),
+ first: FirstOutcome::StreamSuccess,
+ });
+ let routed_client: Arc<dyn RoutedLlmClient> = client;
+ let clients = ClientRouter::new_with_target_prompts(
+ HashMap::from([
+ (ModelId::from("weak"), Arc::clone(&routed_client)),
+ (ModelId::from("strong"), routed_client),
+ ]),
+ HashMap::from([
+ (ModelId::from("weak"), "weak prompt".to_string()),
+ (ModelId::from("strong"), "strong prompt".to_string()),
+ ]),
+ None,
+ );
+
+ let outcome = decide(
+ Arc::new(CandidateAlgorithm {
+ models: vec!["weak".into(), "strong".into()],
+ }),
+ clients,
+ request(),
+ )
+ .await?;
+
+ assert_eq!(outcome.fallback_models, [ModelId::from("strong")]);
+ assert_eq!(instruction_text(&outcome.request), ["weak prompt"]);
+ Ok(())
+ }
+
#[test]
fn fallback_only_accepts_context_and_unavailable_failures() {
let error = |source| LibsyError::client_call("target", source);
diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rs
index dc4db13ee..673a36bc4 100644
--- a/crates/switchyard-runner/src/route.rs
+++ b/crates/switchyard-runner/src/route.rs
@@ -6,7 +6,7 @@
use std::error::Error;
use std::sync::Arc;
-use libsy::{Algorithm, CallModel, LibsyError, RoutingOutcome, drive};
+use libsy::{Algorithm, LibsyError, RoutingOutcome};
use serde_json::Value;
use switchyard_llm_client::{ClientRouter, RunObserver, TranslatingLlmClient};
use switchyard_protocol::{LlmClientError, ModelId, Request, Response, WireFormat};
@@ -194,13 +194,11 @@ impl Route {
})
}
- /// Completes routing-time calls without serving the answer target.
+ /// Completes routing-time calls without serving a post-routing completion.
pub async fn decide(&self, request: Request) -> Result<RoutingOutcome, RunnerError> {
- drive(Arc::clone(&self.algorithm), request, |call| {
- serve_decision_dependency(self.clients.clone(), call)
- })
- .await
- .map_err(Into::into)
+ switchyard_llm_client::decide(Arc::clone(&self.algorithm), self.clients.clone(), request)
+ .await
+ .map_err(Into::into)
}
/// Counts tokens using the configured Anthropic-capable target.
@@ -216,48 +214,3 @@ impl Route {
.map_err(Into::into)
}
}
-
-async fn serve_decision_dependency(clients: ClientRouter, call: CallModel) -> libsy::Result<()> {
- let mut result = Err(LibsyError::NoTargets);
- for (index, model) in call.models.iter().enumerate() {
- // The driver stamps only the first candidate, so every fallback must replace it.
- let mut request = call.request.clone();
- request.llm_request.model = Some(model.to_string());
- let response = match clients.route(model) {
- Ok(client) => client.call(request).await,
- Err(source) => Err(source),
- };
- match response {
- Ok(response) => {
- result = Ok(response);
- break;
- }
- Err(source) => {
- let try_next = index + 1 < call.models.len() && eligible_routing_fallback(&source);
- result = Err(LibsyError::client_call(model.clone(), source));
- if !try_next {
- break;
- }
- }
- }
- }
- call.respond(result)
-}
-
-/// Whether a routing-time candidate failure may fall through to the next model.
-fn eligible_routing_fallback(error: &LlmClientError) -> bool {
- match error {
- LlmClientError::ContextWindowExceeded { .. }
- | LlmClientError::Transport { .. }
- | LlmClientError::Timeout { .. } => true,
- LlmClientError::UpstreamHttp { status, .. } => {
- matches!(
- *status,
- reqwest::StatusCode::FORBIDDEN
- | reqwest::StatusCode::REQUEST_TIMEOUT
- | reqwest::StatusCode::TOO_MANY_REQUESTS
- ) || status.is_server_error()
- }
- _ => false,
- }
-}