Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat: Move libsy::algorithm::run to libsy-llm-client (#324)

Graham King74b899054393
diff --git a/Cargo.lock b/Cargo.lock
index 09354946e..e5e151e0a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2298,7 +2298,6 @@ dependencies = [
"rand 0.10.2",
"serde",
"serde_json",
- "switchyard-llm-client",
"switchyard-protocol",
"thiserror 2.0.18",
"tokio",
@@ -2318,12 +2317,18 @@ dependencies = [
"http",
"httpdate",
"opentelemetry",
+ "opentelemetry_sdk",
+ "parking_lot",
"reqwest",
"serde_json",
+ "switchyard-libsy",
"switchyard-protocol",
"switchyard-translation",
+ "thiserror 2.0.18",
"tokio",
"tracing",
+ "tracing-opentelemetry",
+ "tracing-subscriber",
"wiremock",
]
@@ -2354,6 +2359,7 @@ dependencies = [
"serde_json",
"switchyard-components",
"switchyard-libsy",
+ "switchyard-llm-client",
"switchyard-protocol",
"switchyard-server",
"switchyard-translation",
diff --git a/Cargo.toml b/Cargo.toml
index 90c27d543..0c05c4767 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -45,3 +45,5 @@ switchyard-translation = { path = "crates/switchyard-translation", version = "0.
thiserror = "2"
tokio = { version = "1", features = ["full"] }
tracing = { version = "0.1", default-features = false, features = ["attributes", "std"] }
+tracing-opentelemetry = "0.33"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
diff --git a/crates/libsy-llm-client/Cargo.toml b/crates/libsy-llm-client/Cargo.toml
index 1f5ecc873..359019aef 100644
--- a/crates/libsy-llm-client/Cargo.toml
+++ b/crates/libsy-llm-client/Cargo.toml
@@ -12,10 +12,12 @@ repository.workspace = true
rust-version.workspace = true
[dependencies]
+switchyard-libsy.workspace = true
switchyard-protocol.workspace = true
switchyard-translation.workspace = true
reqwest.workspace = true
async-trait.workspace = true
+futures.workspace = true
futures-util.workspace = true
opentelemetry = { version = "0.32", default-features = false, features = ["metrics"] }
http.workspace = true
@@ -23,7 +25,12 @@ httpdate.workspace = true
serde_json.workspace = true
tokio.workspace = true
tracing.workspace = true
+tracing-opentelemetry.workspace = true
[dev-dependencies]
-futures.workspace = true
+# SDK + in-memory exporter to assert what the observability layer records.
+opentelemetry_sdk = { version = "0.32", features = ["metrics", "testing", "trace"] }
+parking_lot.workspace = true
+thiserror.workspace = true
+tracing-subscriber.workspace = true
wiremock = "0.6"
diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.md
index a1d5bbbc4..1ce9fbf71 100644
--- a/crates/libsy-llm-client/README.md
+++ b/crates/libsy-llm-client/README.md
@@ -11,8 +11,12 @@ encodes the request to that backend's wire format, adds auth and forwards your
headers, makes the call with a shared `reqwest::Client`, and decodes the reply
back into a [`switchyard_protocol::Response`] — buffered or streamed.
-It depends only on `switchyard-protocol` and `switchyard-translation`; no server, no
-provider SDK.
+It also pairs the client with a libsy algorithm: [`run`] drives
+[`Algorithm::run_stream`] and serves every model call the algorithm offloads, so a
+host that just wants the answer never has to drive the step stream itself.
+
+It depends on `switchyard-libsy`, `switchyard-protocol`, and
+`switchyard-translation`; no server, no provider SDK.
## Concepts
@@ -132,6 +136,42 @@ async fn stream(
}
```
+### Routing an algorithm
+
+[`run`] takes a libsy algorithm and a [`ClientRouter`], and returns the final response plus
+the trace of decisions the algorithm published. The router resolves each offloaded call to
+the client for the target the algorithm selected; `ClientRouter::single` is the
+single-provider case:
+
+```rust
+use std::sync::Arc;
+use switchyard_libsy::Algorithm;
+use switchyard_llm_client::{ClientRouter, TranslatingLlmClient};
+use switchyard_protocol::{Context, Request};
+
+async fn route(
+ algorithm: Arc<dyn Algorithm>,
+ client: Arc<TranslatingLlmClient>,
+ request: Request,
+) -> switchyard_libsy::Result<String> {
+ let clients = ClientRouter::single(client);
+ let (trace, _response) =
+ switchyard_llm_client::run(algorithm, clients, Context::default(), request, None).await?;
+ Ok(trace
+ .last()
+ .map(|decision| decision.selected_model().to_string())
+ .unwrap_or_default())
+}
+```
+
+When targets are served by different clients — a judge on one provider and the serving
+models on another, say — build the router from `model name -> client` with
+[`ClientRouter::new`] instead. A model missing from that map fails with
+`LlmClientError::Configuration` rather than silently going to another provider.
+
+A router is not itself a client: [`ClientRouter::route`] hands back a
+[`switchyard_protocol::RoutedLlmClient`] and the caller makes the call.
+
## Cross-format translation
The request/response are translated through the neutral IR, so the inbound shape
diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs
index 920f2f3d6..f1c8a462d 100644
--- a/crates/libsy-llm-client/src/lib.rs
+++ b/crates/libsy-llm-client/src/lib.rs
@@ -10,15 +10,23 @@
//! HTTP call with a shared [`reqwest::Client`], and decodes the wire response
//! back to a [`switchyard_protocol::Response`] — supporting both buffered and
//! streamed responses.
+//!
+//! [`run()`] pairs the client with a libsy algorithm: it drives
+//! [`switchyard_libsy::Algorithm::run_stream`] and serves every model call the algorithm
+//! offloads, so a host that just wants the answer does not have to drive the step stream
+//! itself.
pub mod backend;
pub mod client;
pub mod error;
pub mod metrics;
+mod observability;
pub mod raw;
+pub mod run;
pub use backend::{Backend, DEFAULT_MAX_RETRIES, HttpBackendConfig};
pub use client::{ModelConfig, TranslatingLlmClient};
pub use error::{LlmClientError, Result};
pub use raw::RawResponse;
+pub use run::{ClientRouter, run};
pub use switchyard_translation::RawEventStream;
diff --git a/crates/libsy-llm-client/src/observability.rs b/crates/libsy-llm-client/src/observability.rs
new file mode 100644
index 000000000..1767ffad8
--- /dev/null
+++ b/crates/libsy-llm-client/src/observability.rs
@@ -0,0 +1,335 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! `tracing` span fields for the provider call [`run`](crate::run) makes.
+//!
+//! libsy instruments the orchestration layer (`libsy.run`, `libsy.llm_call`) and owns the
+//! `switchyard.*` metrics. This module owns the other half: the `libsy.client_call` span that
+//! covers the one HTTP call a routed request performs, recorded as OpenTelemetry gen_ai
+//! semantic-convention attributes.
+//!
+//! Nothing here records a metric — these are span fields only, written through the `tracing`
+//! facade so the host's subscriber decides where they go. A streamed response resolves before
+//! its usage is known, so [`observe_client_call`] wraps the stream and keeps the span alive
+//! until it drains, errors, or is abandoned.
+
+use std::borrow::Cow;
+use std::pin::Pin;
+use std::task::{Context as TaskContext, Poll};
+
+use futures::Stream;
+use opentelemetry::{Array as OtelArray, StringValue, Value as OtelValue};
+use switchyard_libsy::{LibsyError, Result};
+use switchyard_protocol::{
+ AggLlmResponse, LlmClientError, LlmRequest, LlmResponse, LlmResponseChunk, LlmResponseStream,
+ LlmResponseStreamEvent, Response, StopReason, Usage,
+};
+use tracing::Span;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
+
+/// Records request parameters represented directly by the neutral IR.
+pub(crate) fn record_gen_ai_request(span: &Span, request: &LlmRequest) {
+ if request.stream {
+ span.record("gen_ai.request.stream", true);
+ }
+ if let Some(value) = request.sampling.temperature {
+ span.record("gen_ai.request.temperature", value);
+ }
+ if let Some(value) = request.sampling.top_p {
+ span.record("gen_ai.request.top_p", value);
+ }
+ if let Some(value) = request.sampling.top_k {
+ span.record("gen_ai.request.top_k", value);
+ }
+ if let Some(value) = request.output.max_output_tokens {
+ span.record("gen_ai.request.max_tokens", otel_int(value));
+ }
+ if let Some(value) = request.reasoning.effort.as_deref() {
+ span.record("gen_ai.request.reasoning.level", value);
+ }
+ if let Some(value) = request
+ .output
+ .response_format
+ .as_ref()
+ .and_then(gen_ai_output_type)
+ {
+ span.record("gen_ai.output.type", value);
+ }
+}
+
+/// Adds terminal response and usage fields to the enclosing `libsy.client_call`
+/// span without consuming or buffering a streaming response.
+pub(crate) fn observe_client_call(result: Result<Response>) -> Result<Response> {
+ let span = Span::current();
+ match result {
+ Ok(mut response) => {
+ match response.llm_response {
+ LlmResponse::Agg(agg) => {
+ span.record("outcome", "ok");
+ record_gen_ai_response(&span, &agg);
+ response.llm_response = LlmResponse::Agg(agg);
+ }
+ LlmResponse::Stream(stream) => {
+ response.llm_response =
+ LlmResponse::Stream(observe_client_stream(stream, span));
+ }
+ }
+ Ok(response)
+ }
+ Err(error) => {
+ let error_type = client_call_error_type(&error);
+ record_client_error(&span, &error_type, &error);
+ Err(error)
+ }
+ }
+}
+
+fn observe_client_stream(stream: LlmResponseStream, span: Span) -> LlmResponseStream {
+ Box::pin(ObservedClientStream {
+ stream,
+ observer: Some(ClientStreamObserver {
+ span,
+ outcome: Outcome::Open,
+ }),
+ })
+}
+
+fn record_gen_ai_response(span: &Span, response: &AggLlmResponse) {
+ record_optional(span, "gen_ai.response.id", response.id.as_deref());
+ record_optional(span, "gen_ai.response.model", response.model.as_deref());
+ record_finish_reasons(
+ span,
+ response
+ .outputs
+ .iter()
+ .filter_map(|output| output.stop_reason)
+ .map(stop_reason_name),
+ );
+ record_gen_ai_usage(span, &response.usage);
+}
+
+fn record_gen_ai_usage(span: &Span, usage: &Usage) {
+ let cache_read = usage.cached_input_tokens();
+ let cache_creation = usage.cache_creation_input_tokens();
+ if usage.input_tokens.is_some() || cache_read.is_some() || cache_creation.is_some() {
+ let input_tokens = usage
+ .input_tokens
+ .unwrap_or_default()
+ .saturating_add(cache_read.unwrap_or_default())
+ .saturating_add(cache_creation.unwrap_or_default());
+ span.record("gen_ai.usage.input_tokens", otel_int(input_tokens));
+ }
+ for (field, value) in [
+ ("gen_ai.usage.output_tokens", usage.output_tokens),
+ ("gen_ai.usage.cache_read.input_tokens", cache_read),
+ ("gen_ai.usage.cache_creation.input_tokens", cache_creation),
+ (
+ "gen_ai.usage.reasoning.output_tokens",
+ usage.reasoning_tokens,
+ ),
+ ] {
+ if let Some(value) = value {
+ span.record(field, otel_int(value));
+ }
+ }
+}
+
+// OpenTelemetry integer attributes are signed; token counts are unsigned in the IR.
+fn otel_int(value: u64) -> i64 {
+ value.min(i64::MAX as u64) as i64
+}
+
+fn record_optional(span: &Span, field: &str, value: Option<&str>) {
+ if let Some(value) = value {
+ span.record(field, value);
+ }
+}
+
+// `tracing` fields cannot preserve a typed string array, so write this attribute directly.
+fn record_finish_reasons(span: &Span, reasons: impl IntoIterator<Item = impl Into<String>>) {
+ let reasons = reasons
+ .into_iter()
+ .map(|reason| StringValue::from(reason.into()))
+ .collect::<Vec<_>>();
+ if !reasons.is_empty() {
+ span.set_attribute(
+ "gen_ai.response.finish_reasons",
+ OtelValue::Array(OtelArray::String(reasons)),
+ );
+ }
+}
+
+fn stop_reason_name(reason: StopReason) -> &'static str {
+ match reason {
+ StopReason::EndTurn => "end_turn",
+ StopReason::MaxTokens => "max_tokens",
+ StopReason::ToolUse => "tool_use",
+ StopReason::ContentFilter => "content_filter",
+ StopReason::Error => "error",
+ StopReason::Unknown => "unknown",
+ }
+}
+
+fn gen_ai_output_type(response_format: &serde_json::Value) -> Option<&'static str> {
+ match response_format
+ .get("type")
+ .and_then(serde_json::Value::as_str)
+ {
+ Some("json" | "json_object" | "json_schema") => Some("json"),
+ Some("text") => Some("text"),
+ _ => None,
+ }
+}
+
+fn client_call_error_type(error: &LibsyError) -> Cow<'static, str> {
+ match error {
+ LibsyError::ClientCall { source, .. } => llm_client_error_type(source),
+ LibsyError::TargetNotFound { .. } => Cow::Borrowed("target_not_found"),
+ LibsyError::NoTargets => Cow::Borrowed("no_targets"),
+ LibsyError::AlgorithmError { .. } => Cow::Borrowed("algorithm_error"),
+ LibsyError::Driver(_) => Cow::Borrowed("driver_error"),
+ LibsyError::AlgorithmTask { .. } => Cow::Borrowed("algorithm_task_error"),
+ LibsyError::MissingFinalResponse => Cow::Borrowed("missing_final_response"),
+ LibsyError::AllTargetsExcluded => Cow::Borrowed("context_window_exceeded"),
+ LibsyError::External { .. } => Cow::Borrowed("_OTHER"),
+ }
+}
+
+fn llm_client_error_type(error: &LlmClientError) -> Cow<'static, str> {
+ match error {
+ LlmClientError::InvalidRequest { .. } => Cow::Borrowed("invalid_request"),
+ LlmClientError::RequestTranslation(_) => Cow::Borrowed("request_translation"),
+ LlmClientError::RequestEncoding(_) => Cow::Borrowed("request_encoding"),
+ LlmClientError::ResponseTranslation(_) => Cow::Borrowed("response_translation"),
+ LlmClientError::Configuration { .. } => Cow::Borrowed("configuration"),
+ LlmClientError::Transport { .. } => Cow::Borrowed("transport"),
+ LlmClientError::Timeout { .. } => Cow::Borrowed("timeout"),
+ LlmClientError::ContextWindowExceeded { .. } => Cow::Borrowed("context_window_exceeded"),
+ LlmClientError::UpstreamHttp { status, .. } => Cow::Owned(status.to_string()),
+ LlmClientError::InvalidResponse { .. } => Cow::Borrowed("invalid_response"),
+ LlmClientError::Ffi { .. } => Cow::Borrowed("ffi"),
+ _ => Cow::Borrowed("_OTHER"),
+ }
+}
+
+// Keep the client span alive until the response is drained, errors, or is abandoned.
+struct ObservedClientStream {
+ stream: LlmResponseStream,
+ observer: Option<ClientStreamObserver>,
+}
+
+impl Stream for ObservedClientStream {
+ type Item = std::result::Result<LlmResponseStreamEvent, LlmClientError>;
+
+ fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Option<Self::Item>> {
+ match self.stream.as_mut().poll_next(cx) {
+ Poll::Ready(Some(item)) => {
+ let failed = self
+ .observer
+ .as_mut()
+ .is_some_and(|observer| observer.observe(&item));
+ if failed {
+ self.observer.take();
+ }
+ Poll::Ready(Some(item))
+ }
+ Poll::Ready(None) => {
+ if let Some(mut observer) = self.observer.take() {
+ observer.complete();
+ }
+ Poll::Ready(None)
+ }
+ Poll::Pending => Poll::Pending,
+ }
+ }
+}
+
+/// What the span has recorded for this stream so far.
+#[derive(PartialEq)]
+enum Outcome {
+ /// Still streaming. Dropping the response now is a cancellation.
+ Open,
+ /// The model finished its message, so `outcome` is already `ok` and dropping the
+ /// response is no longer a cancellation. Observation continues: OpenAI Chat sends usage
+ /// in a chunk *after* the one carrying `finish_reason`, so stopping here would lose it.
+ Completed,
+ /// An error is recorded and terminal; nothing later is worth observing.
+ Failed,
+}
+
+struct ClientStreamObserver {
+ span: Span,
+ outcome: Outcome,
+}
+
+impl ClientStreamObserver {
+ /// Records what `item` says about the response. Returns whether the stream has failed —
+ /// the only state in which the caller should stop observing.
+ fn observe(
+ &mut self,
+ item: &std::result::Result<LlmResponseStreamEvent, LlmClientError>,
+ ) -> bool {
+ match item {
+ Ok(event) => {
+ for chunk in event.normalized() {
+ self.observe_chunk(chunk);
+ if self.outcome == Outcome::Failed {
+ break;
+ }
+ }
+ }
+ Err(error) => {
+ let error_type = llm_client_error_type(error);
+ record_client_error(&self.span, &error_type, error);
+ self.outcome = Outcome::Failed;
+ }
+ }
+ self.outcome == Outcome::Failed
+ }
+
+ fn observe_chunk(&mut self, chunk: &LlmResponseChunk) {
+ match chunk {
+ LlmResponseChunk::MessageStart { id, model } => {
+ record_optional(&self.span, "gen_ai.response.id", id.as_deref());
+ record_optional(&self.span, "gen_ai.response.model", model.as_deref());
+ }
+ LlmResponseChunk::Usage(usage) => record_gen_ai_usage(&self.span, usage),
+ LlmResponseChunk::MessageStop { reason } => {
+ record_finish_reasons(&self.span, reason.iter().cloned());
+ self.complete();
+ }
+ LlmResponseChunk::DecodeError { message } => {
+ record_client_error(&self.span, "response_translation", message);
+ self.outcome = Outcome::Failed;
+ }
+ LlmResponseChunk::StreamError { message } => {
+ record_client_error(&self.span, "502", message);
+ self.outcome = Outcome::Failed;
+ }
+ _ => {}
+ }
+ }
+
+ /// Records the normal terminal outcome, unless an error already claimed it.
+ fn complete(&mut self) {
+ if self.outcome == Outcome::Open {
+ self.span.record("outcome", "ok");
+ self.outcome = Outcome::Completed;
+ }
+ }
+}
+
+impl Drop for ClientStreamObserver {
+ fn drop(&mut self) {
+ if self.outcome == Outcome::Open {
+ self.span.record("outcome", "cancelled");
+ }
+ }
+}
+
+fn record_client_error(span: &Span, error_type: &str, error: &dyn std::fmt::Display) {
+ span.record("outcome", "error");
+ span.record("otel.status_code", "ERROR");
+ span.record("error.type", error_type);
+ span.record("error", tracing::field::display(error));
+}
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
new file mode 100644
index 000000000..efa500ab0
--- /dev/null
+++ b/crates/libsy-llm-client/src/run.rs
@@ -0,0 +1,185 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Drive a libsy algorithm to completion, making the model calls it offloads.
+//!
+//! [`switchyard_libsy::Algorithm::run_stream`] is the whole libsy API: it yields a stream of
+//! steps and expects its consumer to serve every offloaded model call. [`run()`] is that
+//! consumer — it drives the stream with [`switchyard_libsy::drive`], hands each call to a
+//! [`RoutedLlmClient`], and returns the final response with the trace of decisions.
+//!
+//! libsy owns the stream mechanics; what this module adds is the client call itself and the
+//! `libsy.client_call` span around it.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use switchyard_libsy::{
+ Algorithm, CallLlmRequest, LibsyError, Result, RunObserver, algorithm_label, drive,
+};
+use switchyard_protocol::{Context, Decision, LlmClientError, Request, Response, RoutedLlmClient};
+
+use crate::observability;
+
+/// Run one request to completion, serving every offloaded model call with `client`.
+///
+/// Returns the final [`Response`] and the trace of [`Decision`]s the algorithm published along
+/// the way. `observer`, when present, receives each completed model call and, after a
+/// successful routed run, its routing overhead.
+///
+/// `clients` resolves each offloaded call to the client for the target the algorithm
+/// selected — an algorithm may route among targets served by different providers, so this is
+/// a per-call lookup, not one client for the whole run. Use
+/// [`ClientRouter::single`](ClientRouter::single) when one client serves every target.
+///
+/// A failed *model* call is forwarded back into the algorithm, which may route around it;
+/// this returns `Err` only when the run itself cannot complete.
+pub async fn run(
+ algorithm: Arc<dyn Algorithm>,
+ clients: ClientRouter,
+ ctx: Context,
+ request: Request,
+ observer: Option<RunObserver>,
+) -> Result<(Vec<Arc<dyn Decision>>, Response)> {
+ drive(algorithm, ctx, request, observer, move |call| {
+ serve(clients.clone(), call)
+ })
+ .await
+}
+
+/// Serve one offloaded call. A failed *model* call is forwarded to the algorithm via
+/// `respond`; this errors only when the promise itself could not be fulfilled. `serve` makes
+/// the one provider call a routed request performs, so it gets its own `libsy.client_call`
+/// span.
+#[tracing::instrument(
+ target = "libsy",
+ name = "libsy.client_call",
+ skip_all,
+ fields(
+ algorithm = algorithm_label(&call.get_routed().ctx),
+ switchyard.algorithm = algorithm_label(&call.get_routed().ctx),
+ switchyard.routing.tier = tracing::field::Empty,
+ selected_model = call.get_decision().selected_model(),
+ otel.kind = "client",
+ otel.name = %format_args!("chat {}", call.get_decision().selected_model()),
+ openinference.span.kind = "LLM",
+ gen_ai.operation.name = "chat",
+ gen_ai.request.model = call.get_decision().selected_model(),
+ gen_ai.request.stream = tracing::field::Empty,
+ gen_ai.request.temperature = tracing::field::Empty,
+ gen_ai.request.top_p = tracing::field::Empty,
+ gen_ai.request.top_k = tracing::field::Empty,
+ gen_ai.request.max_tokens = tracing::field::Empty,
+ gen_ai.request.reasoning.level = tracing::field::Empty,
+ gen_ai.output.type = tracing::field::Empty,
+ gen_ai.conversation.id = tracing::field::Empty,
+ server.address = tracing::field::Empty,
+ server.port = tracing::field::Empty,
+ gen_ai.response.id = tracing::field::Empty,
+ gen_ai.response.model = tracing::field::Empty,
+ gen_ai.usage.input_tokens = tracing::field::Empty,
+ gen_ai.usage.output_tokens = tracing::field::Empty,
+ gen_ai.usage.cache_read.input_tokens = tracing::field::Empty,
+ gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty,
+ gen_ai.usage.reasoning.output_tokens = tracing::field::Empty,
+ outcome = tracing::field::Empty,
+ otel.status_code = tracing::field::Empty,
+ error.type = tracing::field::Empty,
+ error = tracing::field::Empty,
+ )
+)]
+async fn serve(clients: ClientRouter, call: CallLlmRequest) -> Result<()> {
+ let span = tracing::Span::current();
+ observability::record_gen_ai_request(&span, &call.get_routed().request.llm_request);
+ if let Some(tier) = call.get_decision().routing_tier() {
+ span.record("switchyard.routing.tier", tier);
+ }
+ if let Some(session_id) = call
+ .get_routed()
+ .request
+ .metadata
+ .as_ref()
+ .and_then(|metadata| metadata.session_id.as_deref())
+ {
+ span.record("gen_ai.conversation.id", session_id);
+ }
+ let routed = call.get_routed().clone();
+ let target = routed.decision.selected_model().to_string();
+ let result = match clients.route(&target) {
+ Ok(client) => {
+ client
+ .call(routed.ctx, routed.request, routed.decision)
+ .await
+ }
+ Err(error) => Err(error),
+ }
+ .map_err(|source| LibsyError::client_call(target, source));
+ let result = observability::observe_client_call(result);
+ call.respond(result)
+}
+
+/// Resolves a routed call's selected model to the client that serves it.
+///
+/// An algorithm routes among named targets; which provider each target lives on is the
+/// host's concern, and two targets in one run may sit on different providers. A router owns
+/// that mapping. It is *not* itself a client: it hands back a [`RoutedLlmClient`] and the
+/// caller makes the call.
+///
+/// Cloning is cheap — the mapping is shared, so one router can serve every request.
+#[derive(Clone)]
+pub struct ClientRouter {
+ routing: Arc<Routing>,
+}
+
+enum Routing {
+ /// One client serves every model.
+ Single(Arc<dyn RoutedLlmClient>),
+ /// Each model is served by the client configured for it.
+ ByModel(HashMap<String, Arc<dyn RoutedLlmClient>>),
+}
+
+impl ClientRouter {
+ /// Build a router over `model name -> client`, for targets spread across providers.
+ pub fn new(by_model: HashMap<String, Arc<dyn RoutedLlmClient>>) -> Self {
+ Self {
+ routing: Arc::new(Routing::ByModel(by_model)),
+ }
+ }
+
+ /// A router that serves every model with one client — the single-provider case.
+ ///
+ /// [`TranslatingLlmClient`](crate::TranslatingLlmClient) already maps model names to
+ /// backends internally and rejects ones it does not know, so enumerating them here would
+ /// only duplicate that.
+ pub fn single(client: Arc<dyn RoutedLlmClient>) -> Self {
+ Self {
+ routing: Arc::new(Routing::Single(client)),
+ }
+ }
+
+ /// The client that serves `model`.
+ ///
+ /// Errors with [`LlmClientError::Configuration`] when the router maps models and has no
+ /// entry for this one, rather than silently sending the call to another provider.
+ pub fn route(
+ &self,
+ model: &str,
+ ) -> std::result::Result<&Arc<dyn RoutedLlmClient>, LlmClientError> {
+ match self.routing.as_ref() {
+ Routing::Single(client) => Ok(client),
+ Routing::ByModel(by_model) => {
+ by_model
+ .get(model)
+ .ok_or_else(|| LlmClientError::Configuration {
+ message: format!("no llm client is configured for model {model:?}"),
+ })
+ }
+ }
+ }
+}
+
+impl FromIterator<(String, Arc<dyn RoutedLlmClient>)> for ClientRouter {
+ fn from_iter<I: IntoIterator<Item = (String, Arc<dyn RoutedLlmClient>)>>(iter: I) -> Self {
+ Self::new(iter.into_iter().collect())
+ }
+}
diff --git a/crates/libsy/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs
similarity index 96%
rename from crates/libsy/tests/observability.rs
rename to crates/libsy-llm-client/tests/observability.rs
index 501352f0f..864c95f56 100644
--- a/crates/libsy/tests/observability.rs
+++ b/crates/libsy-llm-client/tests/observability.rs
@@ -34,8 +34,9 @@ use tracing_subscriber::registry::LookupSpan;
use switchyard_libsy::{
Algorithm, Driver, LibsyError, LlmClassifierConfig, LlmTarget, LlmTargetSet, LlmTaskClassifier,
- Step, TaskClassifierConfig,
+ RoutedRequest, Step, TaskClassifierConfig,
};
+use switchyard_llm_client::ClientRouter;
use switchyard_protocol::{
Context, Decision, LlmResponse, Metadata, Request, Response, RoutedLlmClient, Usage,
};
@@ -474,7 +475,11 @@ impl Algorithm for SingleCallAlgo {
});
driver.info(ctx.clone(), decision.clone()).await?;
driver
- .call_llm_target(ctx, &target, request, decision)
+ .call_llm(RoutedRequest {
+ request,
+ decision,
+ ctx,
+ })
.await
}
}
@@ -495,25 +500,39 @@ fn request_with_metadata(session_id: &str, correlation_id: &str) -> Request {
}
}
-fn algo(name: &str, model: &str, client: Option<Arc<dyn RoutedLlmClient>>) -> Arc<dyn Algorithm> {
+fn algo(name: &str, model: &str) -> Arc<dyn Algorithm> {
Arc::new(SingleCallAlgo {
name: name.to_string(),
target_set: LlmTargetSet::new(vec![LlmTarget {
semantic_name: model.to_string(),
- llm_client: client,
}]),
})
}
+/// Drives `algorithm` to completion with `client` serving every target, the way a
+/// single-provider host does.
+async fn run(
+ algorithm: Arc<dyn Algorithm>,
+ client: Arc<dyn RoutedLlmClient>,
+ request: Request,
+) -> switchyard_libsy::Result<(Vec<Arc<dyn Decision>>, Response)> {
+ switchyard_llm_client::run(
+ algorithm,
+ ClientRouter::single(client),
+ Context::default(),
+ request,
+ None,
+ )
+ .await
+}
+
fn classifier_router(
judge_model: &str,
efficient_model: &str,
capable_model: &str,
- client: Arc<dyn RoutedLlmClient>,
) -> switchyard_libsy::Result<Arc<dyn Algorithm>> {
let target = |name: &str| LlmTarget {
semantic_name: name.to_string(),
- llm_client: Some(client.clone()),
};
let targets = LlmTargetSet::new(vec![target(efficient_model), target(capable_model)]);
Ok(Arc::new(LlmTaskClassifier::new(
@@ -612,9 +631,7 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l
request.llm_request.output.max_output_tokens = Some(512);
request.llm_request.output.response_format = Some(json!({"type": "json_schema"}));
request.llm_request.reasoning.effort = Some("high".to_string());
- let (trace, _response) = algo(ALGO, MODEL, Some(client))
- .run(Context::default(), request)
- .await?;
+ let (trace, _response) = run(algo(ALGO, MODEL), client, request).await?;
assert_eq!(trace.len(), 1);
// Metrics: run/call counters and latency histograms keyed by algorithm,
@@ -885,9 +902,7 @@ async fn streamed_usage_updates_the_client_call_span() -> switchyard_libsy::Resu
let client = Arc::new(StreamingUsageClient) as Arc<dyn RoutedLlmClient>;
let mut request = request_with_metadata("obs-stream-session", "obs-stream-corr");
request.llm_request.stream = true;
- let (_, response) = algo(ALGO, MODEL, Some(client))
- .run(Context::default(), request)
- .await?;
+ let (_, response) = run(algo(ALGO, MODEL), client, request).await?;
let LlmResponse::Stream(mut stream) = response.llm_response else {
return Err(test_error("expected a streamed response"));
};
@@ -932,9 +947,7 @@ async fn dropped_stream_records_cancelled_outcome() -> switchyard_libsy::Result<
let client = Arc::new(StreamingUsageClient) as Arc<dyn RoutedLlmClient>;
let mut request = request_with_metadata("obs-cancelled-session", "obs-cancelled-corr");
request.llm_request.stream = true;
- let (_, response) = algo(ALGO, MODEL, Some(client))
- .run(Context::default(), request)
- .await?;
+ let (_, response) = run(algo(ALGO, MODEL), client, request).await?;
let LlmResponse::Stream(stream) = response.llm_response else {
return Err(test_error("expected a streamed response"));
};
@@ -955,13 +968,9 @@ async fn typed_client_failure_records_semantic_error_type() {
let (store, _, _, _, _) = telemetry();
const ALGO: &str = "obs-timeout-algo";
const MODEL: &str = "obs-timeout-model";
- let result = algo(
- ALGO,
- MODEL,
- Some(Arc::new(TimeoutClient) as Arc<dyn RoutedLlmClient>),
- )
- .run(
- Context::default(),
+ let result = run(
+ algo(ALGO, MODEL),
+ Arc::new(TimeoutClient),
request_with_metadata("obs-timeout-session", "obs-timeout-corr"),
)
.await;
@@ -993,8 +1002,8 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::
let total_errors_before =
u64_gauge_value(&before, "switchyard.total_errors").unwrap_or_default();
- // Client-less target: the call is offloaded and we fail it by hand.
- let stream = algo(ALGO, MODEL, None).run_stream(
+ // The call is offloaded and we fail it by hand, without a client.
+ let stream = algo(ALGO, MODEL).run_stream(
Context::default(),
request_with_metadata("obs-session-2", "obs-corr-2"),
None,
@@ -1117,9 +1126,9 @@ async fn classifier_metrics_count_only_the_final_routed_call() -> switchyard_lib
classifier_delay: Duration::from_millis(60),
routed_delay: Duration::from_millis(200),
}) as Arc<dyn RoutedLlmClient>;
- let router = classifier_router("classifier", "weak", "strong", client)?;
+ let router = classifier_router("classifier", "weak", "strong")?;
- let (trace, _response) = router.run(Context::default(), classifier_request()).await?;
+ let (trace, _response) = run(router, client, classifier_request()).await?;
assert_eq!(
trace.last().and_then(|decision| decision.routing_tier()),
@@ -1210,9 +1219,12 @@ async fn classifier_fail_open_records_each_failure_stage() -> switchyard_libsy::
for (judge_model, outcome, expected_reason) in cases {
let client = Arc::new(JudgeClient { outcome }) as Arc<dyn RoutedLlmClient>;
- classifier_router(judge_model, "fo-weak", "fo-strong", client)?
- .run(Context::default(), classifier_request())
- .await?;
+ run(
+ classifier_router(judge_model, "fo-weak", "fo-strong")?,
+ client,
+ classifier_request(),
+ )
+ .await?;
let snapshots = flushed_metrics(exporter, provider);
match expected_reason {
diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.toml
index bea219160..f97d06a80 100644
--- a/crates/libsy/Cargo.toml
+++ b/crates/libsy/Cargo.toml
@@ -33,11 +33,10 @@ thiserror.workspace = true
tokio.workspace = true
tokio-stream = "0.1"
tracing.workspace = true
-tracing-opentelemetry = "0.33"
+tracing-opentelemetry.workspace = true
[dev-dependencies]
# SDK + in-memory exporter to assert what the observability layer records.
opentelemetry_sdk = { version = "0.32", features = ["metrics", "testing", "trace"] }
-switchyard-llm-client.workspace = true
tokio.workspace = true
-tracing-subscriber = "0.3"
+tracing-subscriber.workspace = true
diff --git a/crates/libsy/README.md b/crates/libsy/README.md
index 41907ecfe..9f0f95aa4 100644
--- a/crates/libsy/README.md
+++ b/crates/libsy/README.md
@@ -2,9 +2,9 @@
Provider-neutral orchestration for multi-LLM optimization. A libsy
[`Algorithm`] decides which model targets to call, in what order, and how to
-combine their results. It can use target-owned clients or hand each call back to
-the host, allowing it to embed in proxies, gateways, and agent runtimes without
-owning an HTTP stack.
+combine their results. It hands every call back to the host rather than making
+it, allowing it to embed in proxies, gateways, and agent runtimes without owning
+an HTTP stack.
## Setup
@@ -30,12 +30,15 @@ tokio = { version = "1", features = ["macros", "rt"] }
## How it fits together
-[`LlmTarget`] pairs a semantic routing name with an optional
-[`RoutedLlmClient`](switchyard_protocol::RoutedLlmClient). An [`Algorithm`]
-selects targets and records [`Decision`](switchyard_protocol::Decision)s. Use
-[`Algorithm::run`] with target-owned clients, or [`Algorithm::run_stream`] when
-the host owns model transport. The provider-neutral [`Request`], [`Response`],
-[`Usage`], and [`LlmResponse`] contracts come from `switchyard-protocol`.
+[`LlmTarget`] names a routing destination. An [`Algorithm`] selects targets and
+records [`Decision`](switchyard_protocol::Decision)s, offloading every model call
+to its caller: [`Algorithm::run_stream`] yields a [`Step`] stream whose
+[`Step::CallLlm`] items the host serves over its own transport. libsy makes no
+network calls itself — `switchyard-llm-client`'s `run` is a ready-made consumer
+that drives the stream and performs the calls over HTTP.
+
+The provider-neutral [`Request`], [`Response`], [`Usage`], and [`LlmResponse`]
+contracts come from `switchyard-protocol`.
[`Request`]: switchyard_protocol::Request
[`Response`]: switchyard_protocol::Response
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index 96a7e5093..331943354 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -463,9 +463,9 @@ mod tests {
use crate::core::classifier::Classification;
use crate::{AffinityRouter, SystemPromptProcessor, TargetPrompts};
+ use crate::core::testing::{Serve, drive, echo, reply};
use switchyard_protocol::{
- LlmClientError, LlmRequest, LlmResponse, Message, Metadata, Role, RoutedLlmClient,
- completion_text, text_request, text_response,
+ LlmClientError, LlmRequest, Message, Metadata, Role, completion_text, text_request,
};
#[derive(Debug, thiserror::Error)]
@@ -478,47 +478,15 @@ mod tests {
// --- fixtures ----------------------------------------------------------------------
- /// A client that echoes the routed model name back as the completion.
- struct EchoClient;
-
- #[async_trait]
- impl RoutedLlmClient for EchoClient {
- async fn call(
- &self,
- _ctx: Context,
- _request: Request,
- decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(
- None,
- decision.selected_model().to_string(),
- )),
- metadata: None,
- })
- }
- }
-
- /// A client that captures the request it was handed, so a test can assert on what
- /// actually reached the model.
- struct CapturingClient(Arc<parking_lot::Mutex<Option<Request>>>);
-
- #[async_trait]
- impl RoutedLlmClient for CapturingClient {
- async fn call(
- &self,
- _ctx: Context,
- request: Request,
- decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {
- *self.0.lock() = Some(request);
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(
- None,
- decision.selected_model().to_string(),
- )),
- metadata: None,
- })
+ /// Echoes the routed model name, capturing the request it was handed so a test can
+ /// assert on what actually reached the model.
+ fn capturing(into: Arc<Mutex<Option<Request>>>) -> impl Serve {
+ move |decision: Arc<dyn Decision>, request: Request| {
+ let into = Arc::clone(&into);
+ async move {
+ *into.lock() = Some(request);
+ Ok(reply(decision.selected_model()))
+ }
}
}
@@ -536,35 +504,32 @@ mod tests {
/// Captures the prompt-bearing request that reached the selected target.
#[derive(Default)]
- struct RecordingPromptClient(Mutex<Option<RecordedCall>>);
-
- #[async_trait]
- impl RoutedLlmClient for RecordingPromptClient {
- async fn call(
- &self,
- _ctx: Context,
- request: Request,
- decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {
- *self.0.lock() = Some(RecordedCall {
- target: decision.selected_model().to_string(),
- messages: request
- .llm_request
- .messages
- .iter()
- .filter_map(|message| message.text_content("|"))
- .collect(),
- instructions: request
- .llm_request
- .instructions
- .iter()
- .filter_map(|block| block.content.iter().find_map(text_of))
- .collect(),
- });
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(None, decision.selected_model())),
- metadata: None,
- })
+ struct PromptRecorder(Mutex<Option<RecordedCall>>);
+
+ impl PromptRecorder {
+ fn serve(self: &Arc<Self>) -> impl Serve {
+ let recorder = Arc::clone(self);
+ move |decision: Arc<dyn Decision>, request: Request| {
+ let recorder = Arc::clone(&recorder);
+ async move {
+ *recorder.0.lock() = Some(RecordedCall {
+ target: decision.selected_model().to_string(),
+ messages: request
+ .llm_request
+ .messages
+ .iter()
+ .filter_map(|message| message.text_content("|"))
+ .collect(),
+ instructions: request
+ .llm_request
+ .instructions
+ .iter()
+ .filter_map(|block| block.content.iter().find_map(text_of))
+ .collect(),
+ });
+ Ok(reply(decision.selected_model()))
+ }
+ }
}
}
@@ -575,26 +540,12 @@ mod tests {
}
}
- /// A target set whose targets all serve via [`EchoClient`].
fn target_set(names: &[&str]) -> LlmTargetSet {
LlmTargetSet::new(
names
.iter()
.map(|name| LlmTarget {
semantic_name: name.to_string(),
- llm_client: Some(Arc::new(EchoClient) as Arc<dyn RoutedLlmClient>),
- })
- .collect(),
- )
- }
-
- fn prompt_targets(client: &Arc<RecordingPromptClient>, names: &[&str]) -> LlmTargetSet {
- LlmTargetSet::new(
- names
- .iter()
- .map(|name| LlmTarget {
- semantic_name: (*name).to_string(),
- llm_client: Some(client.clone() as Arc<dyn RoutedLlmClient>),
})
.collect(),
)
@@ -608,20 +559,21 @@ mod tests {
/// Routes one turn on a prompt test cascade and returns the recorded model call.
async fn routed_prompt_call(
- client: &Arc<RecordingPromptClient>,
+ recorder: &Arc<PromptRecorder>,
router: FallThrough,
) -> Result<RecordedCall> {
- Arc::new(router)
- .run(
- Context::default(),
- Request {
- llm_request: text_request(Some("auto".to_string()), "fix the build"),
- raw_request: None,
- metadata: None,
- },
- )
- .await?;
- let call = client.0.lock().take();
+ drive(
+ Arc::new(router),
+ Context::default(),
+ Request {
+ llm_request: text_request(Some("auto".to_string()), "fix the build"),
+ raw_request: None,
+ metadata: None,
+ },
+ recorder.serve(),
+ )
+ .await?;
+ let call = recorder.0.lock().take();
match call {
Some(call) => Ok(call),
None => panic!("the model was never called"),
@@ -629,12 +581,8 @@ mod tests {
}
/// A prompt cascade that always routes to `target`.
- fn prompt_router(
- client: &Arc<RecordingPromptClient>,
- target: &str,
- prompts: TargetPrompts,
- ) -> FallThrough {
- FallThrough::new(prompt_targets(client, &["capable", "efficient"]))
+ fn prompt_router(target: &str, prompts: TargetPrompts) -> FallThrough {
+ FallThrough::new(target_set(&["capable", "efficient"]))
.with_processor(Arc::new(SystemPromptProcessor::new(prompts)))
.with_classifier(Arc::new(DefaultTarget::new(target)))
}
@@ -695,11 +643,12 @@ mod tests {
async fn run_request<S>(
router: &Arc<FallThrough<S>>,
request: Request,
+ serve: impl Serve,
) -> Result<(String, Vec<Arc<dyn Decision>>)>
where
S: Default + Send + 'static,
{
- let (trace, response) = router.clone().run(Context::default(), request).await?;
+ let (trace, response) = drive(router.clone(), Context::default(), request, serve).await?;
let text = response
.llm_response
.into_agg()
@@ -710,16 +659,71 @@ mod tests {
}
/// Drives a shared router through one turn in the default test session.
- async fn run_turn<S>(router: &Arc<FallThrough<S>>) -> Result<(String, Vec<Arc<dyn Decision>>)>
+ async fn run_turn<S>(
+ router: &Arc<FallThrough<S>>,
+ serve: impl Serve,
+ ) -> Result<(String, Vec<Arc<dyn Decision>>)>
where
S: Default + Send + 'static,
{
- run_request(router, request()).await
+ run_request(router, request(), serve).await
+ }
+
+ /// Drives a fresh router through one turn with a specific `serve`.
+ async fn run_with(
+ router: FallThrough,
+ serve: impl Serve,
+ ) -> Result<(String, Vec<Arc<dyn Decision>>)> {
+ run_turn(&Arc::new(router), serve).await
+ }
+
+ /// Discards the recorded calls; the overflow tests that don't assert on them.
+ fn overflows(targets: &'static [&'static str]) -> impl Serve {
+ overflowing(targets, Arc::new(Mutex::new(Vec::new())))
}
- /// Drives a fresh router through one turn.
- async fn run(router: FallThrough) -> Result<(String, Vec<Arc<dyn Decision>>)> {
- run_turn(&Arc::new(router)).await
+ /// Rejects the named `overflowing` targets with a context-window error and echoes for
+ /// the rest, so the retry path can be driven. Every call is recorded in `calls`.
+ fn overflowing(
+ overflowing: &'static [&'static str],
+ calls: Arc<Mutex<Vec<String>>>,
+ ) -> impl Serve {
+ move |decision: Arc<dyn Decision>, _request: Request| {
+ let calls = Arc::clone(&calls);
+ async move {
+ let model = decision.selected_model().to_string();
+ calls.lock().push(model.clone());
+ if overflowing.contains(&model.as_str()) {
+ return Err(LlmClientError::ContextWindowExceeded {
+ model,
+ message: "prompt is too long".to_string(),
+ });
+ }
+ Ok(reply(model))
+ }
+ }
+ }
+
+ /// Rejects the named `unavailable` targets with a 503 and echoes for the rest.
+ /// Every call is recorded in `calls`.
+ fn unavailable(
+ unavailable: &'static [&'static str],
+ calls: Arc<Mutex<Vec<String>>>,
+ ) -> impl Serve {
+ move |decision: Arc<dyn Decision>, _request: Request| {
+ let calls = Arc::clone(&calls);
+ async move {
+ let model = decision.selected_model().to_string();
+ calls.lock().push(model.clone());
+ if unavailable.contains(&model.as_str()) {
+ return Err(LlmClientError::UpstreamHttp {
+ status: 503,
+ body: "unavailable".to_string(),
+ });
+ }
+ Ok(reply(model))
+ }
+ }
}
// --- tests -------------------------------------------------------------------------
@@ -727,10 +731,9 @@ mod tests {
#[tokio::test]
async fn each_target_gets_its_own_prompt() -> Result<()> {
for (target, expected) in [("capable", CAPABLE_PROMPT), ("efficient", EFFICIENT_PROMPT)] {
- let client = Arc::new(RecordingPromptClient::default());
+ let recorder = Arc::new(PromptRecorder::default());
let call =
- routed_prompt_call(&client, prompt_router(&client, target, target_prompts()))
- .await?;
+ routed_prompt_call(&recorder, prompt_router(target, target_prompts())).await?;
assert_eq!(call.target, target);
assert_eq!(call.instructions, vec![expected.to_string()]);
}
@@ -739,11 +742,10 @@ mod tests {
#[tokio::test]
async fn a_target_with_no_prompt_is_left_untouched() -> Result<()> {
- let client = Arc::new(RecordingPromptClient::default());
+ let recorder = Arc::new(PromptRecorder::default());
let only_capable = TargetPrompts::default().with("capable", CAPABLE_PROMPT);
- let call =
- routed_prompt_call(&client, prompt_router(&client, "efficient", only_capable)).await?;
+ let call = routed_prompt_call(&recorder, prompt_router("efficient", only_capable)).await?;
assert!(
call.instructions.is_empty(),
@@ -771,13 +773,13 @@ mod tests {
}
}
- let client = Arc::new(RecordingPromptClient::default());
- let router = FallThrough::new(prompt_targets(&client, &["capable", "efficient"]))
+ let recorder = Arc::new(PromptRecorder::default());
+ let router = FallThrough::new(target_set(&["capable", "efficient"]))
.with_processor(Arc::new(SystemPromptProcessor::new(target_prompts())))
.with_classifier(Arc::new(Abstains))
.with_classifier(Arc::new(DefaultTarget::new("capable")));
- let call = routed_prompt_call(&client, router).await?;
+ let call = routed_prompt_call(&recorder, router).await?;
assert_eq!(call.target, "capable");
assert_eq!(call.instructions, vec![CAPABLE_PROMPT.to_string()]);
@@ -800,137 +802,28 @@ mod tests {
}
}
- let client = Arc::new(RecordingPromptClient::default());
- let router = FallThrough::new(prompt_targets(&client, &["capable", "efficient"]))
+ let recorder = Arc::new(PromptRecorder::default());
+ let router = FallThrough::new(target_set(&["capable", "efficient"]))
.with_processor(Arc::new(Noting))
.with_classifier(Arc::new(DefaultTarget::new("capable")));
- let call = routed_prompt_call(&client, router).await?;
+ let call = routed_prompt_call(&recorder, router).await?;
assert_eq!(call.messages, vec![format!("fix the build|{NOTE}")]);
assert!(call.instructions.is_empty(), "a note is not an instruction");
Ok(())
}
- /// Overflows for the named targets and echoes for the rest, recording every call.
- struct OverflowClient {
- overflowing: Vec<&'static str>,
- calls: Option<Arc<Mutex<Vec<String>>>>,
- }
-
- /// Returns 503 for `weak` and records every routed call.
- struct UnavailableClient(Arc<Mutex<Vec<String>>>);
-
- #[async_trait]
- impl RoutedLlmClient for UnavailableClient {
- async fn call(
- &self,
- _ctx: Context,
- _request: Request,
- decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, LlmClientError> {
- let model = decision.selected_model().to_string();
- self.0.lock().push(model.clone());
- if model == "weak" {
- return Err(LlmClientError::UpstreamHttp {
- status: 503,
- body: "unavailable".to_string(),
- });
- }
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(None, model)),
- metadata: None,
- })
- }
- }
-
- fn unavailable_targets(calls: Arc<Mutex<Vec<String>>>) -> LlmTargetSet {
- let client: Arc<dyn RoutedLlmClient> = Arc::new(UnavailableClient(calls));
- LlmTargetSet::new(
- ["weak", "strong"]
- .into_iter()
- .map(|name| LlmTarget {
- semantic_name: name.to_string(),
- llm_client: Some(Arc::clone(&client)),
- })
- .collect(),
- )
- }
-
- #[async_trait]
- impl RoutedLlmClient for OverflowClient {
- async fn call(
- &self,
- _ctx: Context,
- _request: Request,
- decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, LlmClientError> {
- let model = decision.selected_model().to_string();
- if let Some(calls) = &self.calls {
- calls.lock().push(model.clone());
- }
- if self.overflowing.contains(&model.as_str()) {
- return Err(LlmClientError::ContextWindowExceeded {
- model,
- message: "prompt is too long".to_string(),
- });
- }
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(None, model)),
- metadata: None,
- })
- }
- }
-
- /// `target_set`, but the named `overflowing` targets reject every call with a
- /// context-window error so the retry path can be driven.
- fn target_set_with_overflow(names: &[&str], overflowing: &[&'static str]) -> LlmTargetSet {
- LlmTargetSet::new(
- names
- .iter()
- .map(|name| LlmTarget {
- semantic_name: name.to_string(),
- llm_client: Some(Arc::new(OverflowClient {
- overflowing: overflowing.to_vec(),
- calls: None,
- }) as Arc<dyn RoutedLlmClient>),
- })
- .collect(),
- )
- }
-
- fn counting_overflow_targets(
- names: &[&str],
- overflowing: &[&'static str],
- calls: Arc<Mutex<Vec<String>>>,
- ) -> LlmTargetSet {
- LlmTargetSet::new(
- names
- .iter()
- .map(|name| LlmTarget {
- semantic_name: name.to_string(),
- llm_client: Some(Arc::new(OverflowClient {
- overflowing: overflowing.to_vec(),
- calls: Some(calls.clone()),
- }) as Arc<dyn RoutedLlmClient>),
- })
- .collect(),
- )
- }
-
#[tokio::test]
async fn a_target_that_overflowed_is_skipped_for_the_rest_of_the_session() -> Result<()> {
let calls = Arc::new(Mutex::new(Vec::new()));
let router = Arc::new(
- FallThrough::<()>::new(counting_overflow_targets(
- &["weak", "strong"],
- &["weak"],
- calls.clone(),
- ))
- .with_classifier(fixed(vec![score("weak", 0.9)])),
+ FallThrough::<()>::new(target_set(&["weak", "strong"]))
+ .with_classifier(fixed(vec![score("weak", 0.9)])),
);
for _ in 0..3 {
- assert_eq!(run_turn(&router).await?.0, "strong");
+ let serve = overflowing(&["weak"], calls.clone());
+ assert_eq!(run_turn(&router, serve).await?.0, "strong");
}
assert_eq!(calls.lock().iter().filter(|m| *m == "weak").count(), 1);
Ok(())
@@ -940,20 +833,16 @@ mod tests {
async fn a_different_session_starts_with_an_empty_eviction_set() -> Result<()> {
let calls = Arc::new(Mutex::new(Vec::new()));
let router = Arc::new(
- FallThrough::<()>::new(counting_overflow_targets(
- &["weak", "strong"],
- &["weak"],
- calls.clone(),
- ))
- .with_classifier(fixed(vec![score("weak", 0.9)])),
+ FallThrough::<()>::new(target_set(&["weak", "strong"]))
+ .with_classifier(fixed(vec![score("weak", 0.9)])),
);
- run_turn(&router).await?;
+ run_turn(&router, overflowing(&["weak"], calls.clone())).await?;
let mut other = request();
other.metadata = Some(Metadata {
session_id: Some("session-2".to_string()),
..Metadata::default()
});
- run_request(&router, other).await?;
+ run_request(&router, other, overflowing(&["weak"], calls.clone())).await?;
assert_eq!(calls.lock().iter().filter(|m| *m == "weak").count(), 2);
Ok(())
}
@@ -962,17 +851,13 @@ mod tests {
async fn second_turn_after_full_exhaustion_still_reaches_upstream() -> Result<()> {
let calls = Arc::new(Mutex::new(Vec::new()));
let router = Arc::new(
- FallThrough::<()>::new(counting_overflow_targets(
- &["weak", "strong"],
- &["weak", "strong"],
- calls.clone(),
- ))
- .with_classifier(fixed(vec![score("weak", 0.9)])),
+ FallThrough::<()>::new(target_set(&["weak", "strong"]))
+ .with_classifier(fixed(vec![score("weak", 0.9)])),
);
- let first = run_turn(&router).await;
+ let first = run_turn(&router, overflowing(&["weak", "strong"], calls.clone())).await;
assert!(first.is_err());
calls.lock().clear();
- match run_turn(&router).await {
+ match run_turn(&router, overflowing(&["weak", "strong"], calls.clone())).await {
Err(LibsyError::ClientCall { .. }) => {}
Err(other) => panic!("turn 2 gave {other:?}, calls={:?}", calls.lock()),
Ok(_) => panic!("expected an error"),
@@ -982,10 +867,9 @@ mod tests {
#[tokio::test]
async fn an_overflowing_target_is_retried_on_one_that_fits() -> Result<()> {
- let router =
- FallThrough::<()>::new(target_set_with_overflow(&["weak", "strong"], &["weak"]))
- .with_classifier(fixed(vec![score("weak", 0.9)]));
- let (model, _) = run(router).await?;
+ let router = FallThrough::<()>::new(target_set(&["weak", "strong"]))
+ .with_classifier(fixed(vec![score("weak", 0.9)]));
+ let (model, _) = run_with(router, overflows(&["weak"])).await?;
assert_eq!(model, "strong");
Ok(())
}
@@ -995,14 +879,13 @@ mod tests {
let calls = Arc::new(Mutex::new(Vec::new()));
let affinity = Arc::new(AffinityRouter::new());
let router = Arc::new(
- FallThrough::<()>::new(unavailable_targets(Arc::clone(&calls)))
+ FallThrough::<()>::new(target_set(&["weak", "strong"]))
.with_processor(affinity.clone())
.with_classifier(affinity)
.with_classifier(Arc::new(DefaultTarget::new("weak"))),
);
-
for _ in 0..2 {
- let (model, trace) = run_turn(&router).await?;
+ let (model, trace) = run_turn(&router, unavailable(&["weak"], calls.clone())).await?;
assert_eq!(model, "strong");
assert_eq!(
trace.last().and_then(|decision| decision.fallback_reason()),
@@ -1015,12 +898,9 @@ mod tests {
#[tokio::test]
async fn overflowing_targets_are_retried_until_one_fits() -> Result<()> {
- let router = FallThrough::<()>::new(target_set_with_overflow(
- &["weak", "mid", "strong"],
- &["weak", "mid"],
- ))
- .with_classifier(fixed(vec![score("weak", 0.9)]));
- let (model, _) = run(router).await?;
+ let router = FallThrough::<()>::new(target_set(&["weak", "mid", "strong"]))
+ .with_classifier(fixed(vec![score("weak", 0.9)]));
+ let (model, _) = run_with(router, overflows(&["weak", "mid"])).await?;
assert_eq!(model, "strong");
Ok(())
}
@@ -1028,12 +908,9 @@ mod tests {
#[tokio::test]
async fn exhausting_every_target_surfaces_the_client_overflow() -> Result<()> {
// Only the client error maps to a 400 upstream, so it must survive exhaustion.
- let router = FallThrough::<()>::new(target_set_with_overflow(
- &["weak", "strong"],
- &["weak", "strong"],
- ))
- .with_classifier(fixed(vec![score("weak", 0.9)]));
- match run(router).await {
+ let router = FallThrough::<()>::new(target_set(&["weak", "strong"]))
+ .with_classifier(fixed(vec![score("weak", 0.9)]));
+ match run_with(router, overflows(&["weak", "strong"])).await {
Ok(_) => panic!("expected an overflow error, got a response"),
Err(LibsyError::ClientCall {
source: LlmClientError::ContextWindowExceeded { .. },
@@ -1062,11 +939,10 @@ mod tests {
}
let seen = Arc::new(Mutex::new(Vec::new()));
- let router =
- FallThrough::<()>::new(target_set_with_overflow(&["weak", "strong"], &["weak"]))
- .with_classifier(fixed(vec![score("weak", 0.9)]))
- .with_processor(Arc::new(CountingProcessor(seen.clone())));
- let (model, _) = run(router).await?;
+ let router = FallThrough::<()>::new(target_set(&["weak", "strong"]))
+ .with_classifier(fixed(vec![score("weak", 0.9)]))
+ .with_processor(Arc::new(CountingProcessor(seen.clone())));
+ let (model, _) = run_with(router, overflows(&["weak"])).await?;
assert_eq!(model, "strong");
assert_eq!(seen.lock().iter().filter(|e| **e == "request").count(), 1);
assert_eq!(seen.lock().iter().filter(|e| **e == "decision").count(), 1);
@@ -1082,7 +958,7 @@ mod tests {
);
let mut ctx = Context::default();
ctx.exclude_target("weak");
- let (trace, response) = router.run(ctx, request()).await?;
+ let (trace, response) = drive(router, ctx, request(), echo()).await?;
let text = response
.llm_response
.into_agg()
@@ -1104,7 +980,7 @@ mod tests {
async fn argmax_picks_the_highest_confidence_target() -> Result<()> {
let router = FallThrough::<()>::new(target_set(&["strong", "weak"]))
.with_classifier(fixed(vec![score("weak", 0.2), score("strong", 0.9)]));
- let (model, trace) = run(router).await?;
+ let (model, trace) = run_with(router, echo()).await?;
assert_eq!(model, "strong");
assert_eq!(trace.len(), 1);
assert_eq!(trace[0].selected_model(), "strong");
@@ -1117,7 +993,7 @@ mod tests {
let router = FallThrough::<()>::new(target_set(&["strong", "weak"]))
.with_classifier(fixed(vec![]))
.with_classifier(fixed(vec![score("weak", 1.0)]));
- let (model, _) = run(router).await?;
+ let (model, _) = run_with(router, echo()).await?;
assert_eq!(model, "weak");
Ok(())
}
@@ -1128,7 +1004,7 @@ mod tests {
let router = FallThrough::<()>::new(target_set(&["strong", "weak"]))
.with_classifier(fixed(vec![score("strong", 0.6)]))
.with_classifier(fixed(vec![score("weak", 1.0)]));
- let (model, _) = run(router).await?;
+ let (model, _) = run_with(router, echo()).await?;
assert_eq!(model, "strong");
Ok(())
}
@@ -1137,7 +1013,7 @@ mod tests {
async fn all_abstaining_is_an_error() -> Result<()> {
let router =
FallThrough::<()>::new(target_set(&["strong", "weak"])).with_classifier(fixed(vec![]));
- let error = run(router)
+ let error = run_with(router, echo())
.await
.err()
.ok_or_else(|| test_error("expected classifiers to abstain"))?;
@@ -1171,7 +1047,7 @@ mod tests {
let router = FallThrough::<()>::new(target_set(&["strong", "weak"]))
.with_classifier(Arc::new(NeedsDriver));
- let (model, _) = run(router).await?;
+ let (model, _) = run_with(router, echo()).await?;
assert_eq!(model, "strong");
Ok(())
}
@@ -1201,7 +1077,7 @@ mod tests {
let router = FallThrough::<()>::new(target_set(&["strong", "weak"]))
.with_processor(Arc::new(RecordingProcessor(seen.clone())))
.with_classifier(fixed(vec![score("strong", 1.0)]));
- run(router).await?;
+ run_with(router, echo()).await?;
assert_eq!(*seen.lock(), vec!["request", "decision"]);
Ok(())
@@ -1255,16 +1131,13 @@ mod tests {
let seen_by_classifier = Arc::new(Mutex::new(Vec::new()));
let seen_by_model = Arc::new(Mutex::new(None));
- let targets = LlmTargetSet::new(vec![LlmTarget {
- semantic_name: "strong".to_string(),
- llm_client: Some(Arc::new(CapturingClient(seen_by_model.clone()))),
- }]);
+ let targets = target_set(&["strong"]);
let router = FallThrough::new(targets)
.with_processor(Arc::new(Appender("first")))
.with_processor(Arc::new(Appender("second")))
.with_classifier(Arc::new(TrailClassifier(seen_by_classifier.clone())));
- run_turn(&Arc::new(router)).await?;
+ run_turn(&Arc::new(router), capturing(seen_by_model.clone())).await?;
// The classifier saw both processors' edits, in chain order, on top of the original.
assert_eq!(*seen_by_classifier.lock(), vec!["hi", "first", "second"]);
@@ -1326,8 +1199,8 @@ mod tests {
.with_classifier(Arc::new(ThresholdClassifier)),
);
- let (turn1, _) = run_turn(&router).await?;
- let (turn2, _) = run_turn(&router).await?;
+ let (turn1, _) = run_turn(&router, echo()).await?;
+ let (turn2, _) = run_turn(&router, echo()).await?;
let final_request = Request {
metadata: Some(Metadata {
session_id: Some("session-1".to_string()),
@@ -1336,8 +1209,8 @@ mod tests {
}),
..request()
};
- let (final_turn, _) = run_request(&router, final_request).await?;
- let (restarted_session, _) = run_turn(&router).await?;
+ let (final_turn, _) = run_request(&router, final_request, echo()).await?;
+ let (restarted_session, _) = run_turn(&router, echo()).await?;
let (second_session, _) = run_request(
&router,
Request {
@@ -1347,14 +1220,15 @@ mod tests {
}),
..request()
},
+ echo(),
)
.await?;
let anonymous = Request {
metadata: None,
..request()
};
- let (anonymous1, _) = run_request(&router, anonymous.clone()).await?;
- let (anonymous2, _) = run_request(&router, anonymous).await?;
+ let (anonymous1, _) = run_request(&router, anonymous.clone(), echo()).await?;
+ let (anonymous2, _) = run_request(&router, anonymous, echo()).await?;
assert_eq!(turn1, "weak");
assert_eq!(turn2, "strong");
@@ -1378,7 +1252,7 @@ mod tests {
..request()
};
- let result = router.clone().run(Context::default(), final_request).await;
+ let result = drive(router.clone(), Context::default(), final_request, echo()).await;
assert!(matches!(result, Err(LibsyError::AlgorithmError { .. })));
let states = router
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index 95f50d312..d75748d14 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -21,7 +21,7 @@ use super::util::llm_judge::{
SerdeDecoder, StructuredJudge,
};
use super::util::target_selector::TargetSelectorPolicy;
-use crate::core::algorithm::{Algorithm, Driver, LlmTarget, LlmTargetSet};
+use crate::core::algorithm::{Algorithm, Driver, LlmTarget, LlmTargetSet, RoutedRequest};
use crate::core::classifier::{Classification, Classifier, Score};
use crate::core::state::{State, StateValue};
use crate::{LibsyError, Result};
@@ -519,15 +519,14 @@ impl Classifier<State> for EscalationClassifier {
// `(decisive(capable), None)` tells FallThrough::execute to call
// call_llm_with_fallback with the capable target instead of surfacing the error.
let efficient_response = match driver
- .call_llm_target(
- Context::default(),
- &self.efficient,
- request.clone(),
- Arc::new(SimpleDecision {
+ .call_llm(RoutedRequest {
+ request: request.clone(),
+ decision: Arc::new(SimpleDecision {
selected_model: self.efficient.semantic_name.clone(),
reasoning: Some("escalation classifier: efficient tier".into()),
}),
- )
+ ctx: Context::default(),
+ })
.await
{
Ok(r) => r,
@@ -960,8 +959,8 @@ mod tests {
};
use crate::algorithms::util::llm_judge::Judge;
- use crate::core::algorithm::Algorithm;
- use switchyard_protocol::{Context, LlmResponse, Response, RoutedLlmClient};
+ use crate::core::testing::{Serve, drive, reply};
+ use switchyard_protocol::{Context, LlmResponse, Response};
const TEST_THRESHOLD: f64 = 0.5;
@@ -1002,14 +1001,16 @@ mod tests {
})
}
+ /// Records what each target received; answers the judge with a supported verdict and
+ /// every other target with a plain completion.
#[derive(Default)]
- struct PerRequestClient {
+ struct Recorder {
calls: Mutex<Vec<String>>,
judge_max_output_tokens: Mutex<Vec<Option<u64>>>,
judge_system_prompts: Mutex<Vec<String>>,
}
- impl PerRequestClient {
+ impl Recorder {
fn calls(&self) -> Vec<String> {
self.calls.lock().clone()
}
@@ -1021,58 +1022,50 @@ mod tests {
fn judge_system_prompts(&self) -> Vec<String> {
self.judge_system_prompts.lock().clone()
}
- }
- #[async_trait]
- impl RoutedLlmClient for PerRequestClient {
- async fn call(
- &self,
- _ctx: Context,
- request: Request,
- decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, LlmClientError> {
- let model = decision.selected_model().to_string();
- self.calls.lock().push(model.clone());
- let completion = if model == "judge" {
- self.judge_max_output_tokens
- .lock()
- .push(request.llm_request.output.max_output_tokens);
- self.judge_system_prompts.lock().extend(
- request
- .llm_request
- .instructions
- .first()
- .and_then(|instruction| {
- instruction.content.iter().find_map(|b| {
- if let ContentBlock::Text { text } = b {
- Some(text.clone())
- } else {
- None
- }
- })
- }),
- );
- r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#.to_string()
- } else {
- format!("answer from {model}")
- };
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(None, completion)),
- metadata: request.metadata,
- })
+ fn serve(self: &Arc<Self>) -> impl Serve {
+ let recorder = Arc::clone(self);
+ move |decision: Arc<dyn Decision>, request: Request| {
+ let recorder = Arc::clone(&recorder);
+ async move {
+ let model = decision.selected_model().to_string();
+ recorder.calls.lock().push(model.clone());
+ let completion = if model == "judge" {
+ recorder
+ .judge_max_output_tokens
+ .lock()
+ .push(request.llm_request.output.max_output_tokens);
+ recorder.judge_system_prompts.lock().extend(
+ request
+ .llm_request
+ .instructions
+ .first()
+ .and_then(|instruction| {
+ instruction.content.iter().find_map(|b| {
+ if let ContentBlock::Text { text } = b {
+ Some(text.clone())
+ } else {
+ None
+ }
+ })
+ }),
+ );
+ r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#.to_string()
+ } else {
+ format!("answer from {model}")
+ };
+ Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(None, completion)),
+ metadata: request.metadata,
+ })
+ }
+ }
}
}
- struct UnreachableJudgeClient;
-
- #[async_trait]
- impl RoutedLlmClient for UnreachableJudgeClient {
- async fn call(
- &self,
- _ctx: Context,
- request: Request,
- decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, LlmClientError> {
+ /// The judge times out; every other target answers normally.
+ fn unreachable_judge() -> impl Serve {
+ |decision: Arc<dyn Decision>, request: Request| async move {
let model = decision.selected_model().to_string();
if model == "judge" {
return Err(LlmClientError::Timeout {
@@ -1086,10 +1079,9 @@ mod tests {
}
}
- fn router(client: Arc<dyn RoutedLlmClient>) -> Result<Arc<LlmTaskClassifier>> {
+ fn router() -> Result<Arc<LlmTaskClassifier>> {
let target = |name: &str| LlmTarget {
semantic_name: name.to_string(),
- llm_client: Some(client.clone()),
};
Ok(Arc::new(LlmTaskClassifier::new(
LlmClassifierConfig::Capability {
@@ -1134,9 +1126,15 @@ mod tests {
#[tokio::test]
async fn an_unreachable_judge_routes_capable_instead_of_failing_the_request() -> Result<()> {
- let router = router(Arc::new(UnreachableJudgeClient))?;
+ let router = router()?;
- let (trace, response) = router.run(Context::default(), classify_request()).await?;
+ let (trace, response) = drive(
+ router,
+ Context::default(),
+ classify_request(),
+ unreachable_judge(),
+ )
+ .await?;
assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));
assert_eq!(
@@ -1148,15 +1146,27 @@ mod tests {
#[tokio::test]
async fn classifier_judges_each_request_without_affinity() -> Result<()> {
- let client = Arc::new(PerRequestClient::default());
- let router = router(client.clone())?;
+ let recorder = Arc::new(Recorder::default());
+ let router = router()?;
let request = classify_request;
- router.clone().run(Context::default(), request()).await?;
- router.clone().run(Context::default(), request()).await?;
+ drive(
+ router.clone(),
+ Context::default(),
+ request(),
+ recorder.serve(),
+ )
+ .await?;
+ drive(
+ router.clone(),
+ Context::default(),
+ request(),
+ recorder.serve(),
+ )
+ .await?;
assert_eq!(
- client.calls(),
+ recorder.calls(),
vec!["judge", "efficient", "judge", "efficient"]
);
Ok(())
@@ -1164,10 +1174,9 @@ mod tests {
#[tokio::test]
async fn classifier_config_sets_the_judge_completion_cap() -> Result<()> {
- let client = Arc::new(PerRequestClient::default());
+ let recorder = Arc::new(Recorder::default());
let target = |name: &str| LlmTarget {
semantic_name: name.to_string(),
- llm_client: Some(client.clone()),
};
let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {
judge_target: target("judge"),
@@ -1179,18 +1188,23 @@ mod tests {
},
})?);
- router.run(Context::default(), classify_request()).await?;
+ drive(
+ router,
+ Context::default(),
+ classify_request(),
+ recorder.serve(),
+ )
+ .await?;
- assert_eq!(client.judge_max_output_tokens(), vec![Some(512)]);
+ assert_eq!(recorder.judge_max_output_tokens(), vec![Some(512)]);
Ok(())
}
#[tokio::test]
async fn classifier_config_overrides_the_packaged_prompt() -> Result<()> {
- let client = Arc::new(PerRequestClient::default());
+ let recorder = Arc::new(Recorder::default());
let target = |name: &str| LlmTarget {
semantic_name: name.to_string(),
- llm_client: Some(client.clone()),
};
let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {
judge_target: target("judge"),
@@ -1203,9 +1217,15 @@ mod tests {
},
})?);
- router.run(Context::default(), classify_request()).await?;
+ drive(
+ router,
+ Context::default(),
+ classify_request(),
+ recorder.serve(),
+ )
+ .await?;
- let prompts = client.judge_system_prompts();
+ let prompts = recorder.judge_system_prompts();
assert_eq!(prompts.len(), 1);
assert_eq!(prompts[0], "Custom capability rubric.");
Ok(())
@@ -1213,10 +1233,9 @@ mod tests {
#[tokio::test]
async fn classifier_config_enables_session_affinity() -> Result<()> {
- let client = Arc::new(PerRequestClient::default());
+ let recorder = Arc::new(Recorder::default());
let target = |name: &str| LlmTarget {
semantic_name: name.to_string(),
- llm_client: Some(client.clone()),
};
let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {
judge_target: target("judge"),
@@ -1228,25 +1247,31 @@ mod tests {
},
})?);
- router
- .clone()
- .run(Context::default(), classify_session_request())
- .await?;
- router
- .clone()
- .run(Context::default(), classify_session_request())
- .await?;
+ let session_request = classify_session_request;
+ drive(
+ router.clone(),
+ Context::default(),
+ session_request(),
+ recorder.serve(),
+ )
+ .await?;
+ drive(
+ router.clone(),
+ Context::default(),
+ session_request(),
+ recorder.serve(),
+ )
+ .await?;
- assert_eq!(client.calls(), vec!["judge", "efficient", "efficient"]);
+ assert_eq!(recorder.calls(), vec!["judge", "efficient", "efficient"]);
Ok(())
}
#[tokio::test]
async fn classifier_config_reuses_message_hash_affinity_for_a_follow_up() -> Result<()> {
- let client = Arc::new(PerRequestClient::default());
+ let recorder = Arc::new(Recorder::default());
let target = |name: &str| LlmTarget {
semantic_name: name.to_string(),
- llm_client: Some(client.clone()),
};
let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {
judge_target: target("judge"),
@@ -1260,16 +1285,22 @@ mod tests {
},
})?);
- router
- .clone()
- .run(Context::default(), classify_request())
- .await?;
- router
- .clone()
- .run(Context::default(), classify_follow_up_request())
- .await?;
+ drive(
+ router.clone(),
+ Context::default(),
+ classify_request(),
+ recorder.serve(),
+ )
+ .await?;
+ drive(
+ router.clone(),
+ Context::default(),
+ classify_follow_up_request(),
+ recorder.serve(),
+ )
+ .await?;
- assert_eq!(client.calls(), vec!["judge", "efficient", "efficient"]);
+ assert_eq!(recorder.calls(), vec!["judge", "efficient", "efficient"]);
Ok(())
}
@@ -1313,7 +1344,6 @@ mod tests {
fn invalid_classifier_config_is_rejected() -> Result<()> {
let target = |name: &str| LlmTarget {
semantic_name: name.to_string(),
- llm_client: None,
};
for bad in [1.5, -0.1, f64::NAN, f64::INFINITY] {
assert!(
@@ -1765,57 +1795,54 @@ mod tests {
use std::collections::VecDeque;
- use switchyard_protocol::LlmClientError as ClientError;
-
use switchyard_protocol::Decision;
- /// Serves each call with the next queued reply.
- struct QueuedClient {
- replies: Mutex<VecDeque<String>>,
- }
+ /// A queue of replies, drained in order.
+ struct Queue(Mutex<VecDeque<String>>);
- impl QueuedClient {
+ impl Queue {
fn new(replies: impl IntoIterator<Item = &'static str>) -> Arc<Self> {
- Arc::new(Self {
- replies: Mutex::new(replies.into_iter().map(String::from).collect()),
- })
+ Arc::new(Self(Mutex::new(
+ replies.into_iter().map(String::from).collect(),
+ )))
}
- }
- #[async_trait]
- impl RoutedLlmClient for QueuedClient {
- async fn call(
- &self,
- _ctx: Context,
- request: Request,
- _decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, ClientError> {
- let reply = self
- .replies
+ fn take(&self) -> String {
+ self.0
.lock()
.pop_front()
- .unwrap_or_else(|| "unexpected call".to_string());
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(None, reply)),
- metadata: request.metadata,
- })
+ .unwrap_or_else(|| "unexpected call".to_string())
+ }
+ }
+
+ /// Serves the judge target from `judge` and every other target from `model`, each with
+ /// its next queued reply.
+ fn queued(model: Arc<Queue>, judge: Arc<Queue>) -> impl Serve {
+ move |decision: Arc<dyn Decision>, request: Request| {
+ let queue = if decision.selected_model() == "judge" {
+ Arc::clone(&judge)
+ } else {
+ Arc::clone(&model)
+ };
+ async move {
+ Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(None, queue.take())),
+ metadata: request.metadata,
+ })
+ }
}
}
/// Builds a router with escalation enabled (`confirmations=1` latches on the first verdict).
- fn escalation_router(
- client: Arc<dyn RoutedLlmClient>,
- judge_client: Arc<dyn RoutedLlmClient>,
- ) -> Result<Arc<LlmTaskClassifier>> {
- let target = |name: &str, c: Arc<dyn RoutedLlmClient>| LlmTarget {
+ fn escalation_router() -> Result<Arc<LlmTaskClassifier>> {
+ let target = |name: &str| LlmTarget {
semantic_name: name.to_string(),
- llm_client: Some(c),
};
Ok(Arc::new(LlmTaskClassifier::new(
LlmClassifierConfig::Escalation {
- judge_target: target("judge", judge_client),
- efficient_target: target("efficient", client.clone()),
- capable_target: target("capable", client),
+ judge_target: target("judge"),
+ efficient_target: target("efficient"),
+ capable_target: target("capable"),
contract: ClassifierContractConfig::default(),
config: EscalationJudgeConfig {
confirmations: 1,
@@ -1829,12 +1856,17 @@ mod tests {
#[tokio::test]
async fn escalation_router_serves_efficient_when_judge_declines() -> Result<()> {
// Judge: no escalation. Expect the efficient response to be returned directly.
- let judge_client = QueuedClient::new([r#"{"escalate":false,"reason":"progressing"}"#]);
- let model_client = QueuedClient::new(["efficient answer"]);
- let router = escalation_router(model_client, judge_client)?;
- let request = classify_request();
-
- let (trace, response) = router.run(Context::default(), request).await?;
+ let judge = Queue::new([r#"{"escalate":false,"reason":"progressing"}"#]);
+ let model = Queue::new(["efficient answer"]);
+ let router = escalation_router()?;
+
+ let (trace, response) = drive(
+ router,
+ Context::default(),
+ classify_request(),
+ queued(model, judge),
+ )
+ .await?;
// The efficient model is the serving target, and the response comes from its call.
assert_eq!(trace.last().map(|d| d.selected_model()), Some("efficient"));
@@ -1847,10 +1879,9 @@ mod tests {
#[tokio::test]
async fn escalation_config_overrides_the_packaged_prompt() -> Result<()> {
- let client = Arc::new(PerRequestClient::default());
+ let recorder = Arc::new(Recorder::default());
let target = |name: &str| LlmTarget {
semantic_name: name.to_string(),
- llm_client: Some(client.clone()),
};
let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Escalation {
judge_target: target("judge"),
@@ -1864,9 +1895,15 @@ mod tests {
max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,
})?);
- router.run(Context::default(), classify_request()).await?;
+ drive(
+ router,
+ Context::default(),
+ classify_request(),
+ recorder.serve(),
+ )
+ .await?;
- let prompts = client.judge_system_prompts();
+ let prompts = recorder.judge_system_prompts();
assert_eq!(prompts.len(), 1);
assert_eq!(prompts[0], "Custom trajectory rubric.");
Ok(())
@@ -1875,13 +1912,18 @@ mod tests {
#[tokio::test]
async fn escalation_router_upgrades_to_capable_when_judge_escalates() -> Result<()> {
// Judge: escalate. After the efficient call, the streak confirms and capable is served.
- let judge_client = QueuedClient::new([r#"{"escalate":true,"reason":"stuck in a loop"}"#]);
+ let judge = Queue::new([r#"{"escalate":true,"reason":"stuck in a loop"}"#]);
// Efficient is called first (by the classifier), then capable is called by FallThrough.
- let model_client = QueuedClient::new(["efficient draft", "capable answer"]);
- let router = escalation_router(model_client, judge_client)?;
- let request = classify_request();
-
- let (trace, response) = router.run(Context::default(), request).await?;
+ let model = Queue::new(["efficient draft", "capable answer"]);
+ let router = escalation_router()?;
+
+ let (trace, response) = drive(
+ router,
+ Context::default(),
+ classify_request(),
+ queued(model, judge),
+ )
+ .await?;
assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));
assert_eq!(
@@ -1895,19 +1937,25 @@ mod tests {
async fn escalation_router_stays_capable_after_latch() -> Result<()> {
// First turn: judge escalates and the streak latches.
// Second turn: judge is not called again; capable is served directly.
- let judge_client = QueuedClient::new([r#"{"escalate":true,"reason":"stuck"}"#]);
- let model_client = QueuedClient::new(["efficient draft", "capable t1", "capable t2"]);
- let router = escalation_router(model_client, judge_client)?;
+ let judge = Queue::new([r#"{"escalate":true,"reason":"stuck"}"#]);
+ let model = Queue::new(["efficient draft", "capable t1", "capable t2"]);
+ let router = escalation_router()?;
let session_request = classify_session_request();
- router
- .clone()
- .run(Context::default(), session_request.clone())
- .await?;
- let (trace, _) = router
- .clone()
- .run(Context::default(), session_request)
- .await?;
+ drive(
+ router.clone(),
+ Context::default(),
+ session_request.clone(),
+ queued(Arc::clone(&model), Arc::clone(&judge)),
+ )
+ .await?;
+ let (trace, _) = drive(
+ router.clone(),
+ Context::default(),
+ session_request,
+ queued(model, judge),
+ )
+ .await?;
assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));
Ok(())
@@ -1918,39 +1966,22 @@ mod tests {
// When the efficient model exceeds its context window inside score(), the classifier
// must return capable rather than propagating the error — otherwise the client sees
// HTTP 400 instead of a response from the strong model.
- struct OverflowClient;
- #[async_trait]
- impl RoutedLlmClient for OverflowClient {
- async fn call(
- &self,
- _ctx: Context,
- _request: Request,
- decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, LlmClientError> {
- Err(LlmClientError::ContextWindowExceeded {
+ let router = escalation_router()?;
+
+ // Efficient overflows, capable answers, and the judge must never be called.
+ let serve = |decision: Arc<dyn Decision>, _request: Request| async move {
+ match decision.selected_model() {
+ "efficient" => Err(LlmClientError::ContextWindowExceeded {
model: decision.selected_model().to_string(),
message: "prompt is too long".to_string(),
- })
+ }),
+ "judge" => panic!("the judge must not be consulted when efficient overflows"),
+ _ => Ok(reply("capable answer")),
}
- }
-
- let target = |name: &str, c: Arc<dyn RoutedLlmClient>| LlmTarget {
- semantic_name: name.to_string(),
- llm_client: Some(c),
};
- let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Escalation {
- judge_target: target("judge", QueuedClient::new([])), // must not be called
- efficient_target: target("efficient", Arc::new(OverflowClient)),
- capable_target: target("capable", QueuedClient::new(["capable answer"])),
- contract: ClassifierContractConfig::default(),
- config: EscalationJudgeConfig {
- confirmations: 1,
- ..EscalationJudgeConfig::default()
- },
- max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,
- })?);
- let (trace, response) = router.run(Context::default(), classify_request()).await?;
+ let (trace, response) =
+ drive(router, Context::default(), classify_request(), serve).await?;
assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));
assert_eq!(
diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rs
index 71118cd4f..66e2b9cdb 100644
--- a/crates/libsy/src/algorithms/noop.rs
+++ b/crates/libsy/src/algorithms/noop.rs
@@ -79,6 +79,7 @@ mod tests {
use switchyard_protocol::{LlmRequest, Message, Role};
use super::*;
+ use crate::core::testing::{drive, echo};
#[tokio::test]
async fn test_noop_algo() -> Result<()> {
@@ -93,8 +94,10 @@ mod tests {
metadata: None,
};
+ // `Noop` synthesizes its own response and never offloads a call, so `echo` is
+ // never reached.
let a: Arc<dyn Algorithm> = Arc::new(Noop {});
- let (decisions, response) = a.run(Context::default(), request).await?;
+ let (decisions, response) = drive(a, Context::default(), request, echo()).await?;
let Some(decision) = decisions.first() else {
panic!("Expected exactly one Decision");
};
diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rs
index 9643fc4b4..85817ba4d 100644
--- a/crates/libsy/src/algorithms/passthrough.rs
+++ b/crates/libsy/src/algorithms/passthrough.rs
@@ -8,7 +8,7 @@ use std::sync::Arc;
use switchyard_protocol::{Request, Response};
use crate::Result;
-use crate::core::algorithm::{Algorithm, Driver, LlmTarget};
+use crate::core::algorithm::{Algorithm, Driver, LlmTarget, RoutedRequest};
use switchyard_protocol::{Context, Decision};
/// Routing algorithm that always calls one configured target.
@@ -58,7 +58,11 @@ impl Algorithm for Passthrough {
});
driver.info(ctx.clone(), decision.clone()).await?;
driver
- .call_llm_target(ctx, &self.target, request, decision)
+ .call_llm(RoutedRequest {
+ request,
+ decision,
+ ctx,
+ })
.await
}
}
@@ -69,29 +73,8 @@ mod tests {
use super::Passthrough;
use crate::core::algorithm::{Algorithm, LlmTarget};
- use switchyard_protocol::{
- Context, Decision, LlmResponse, Request, Response, RoutedLlmClient, completion_text,
- text_request, text_response,
- };
-
- /// Echoes the selected target so tests can inspect which target was called.
- /// TODO: Duplicated from rand.rs
- struct EchoClient;
-
- #[async_trait::async_trait]
- impl RoutedLlmClient for EchoClient {
- async fn call(
- &self,
- _ctx: Context,
- _request: Request,
- decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(None, decision.selected_model())),
- metadata: None,
- })
- }
- }
+ use crate::core::testing::{drive, echo};
+ use switchyard_protocol::{Context, Request, completion_text, text_request};
#[tokio::test]
async fn test_passthrough() -> crate::Result<()> {
@@ -103,9 +86,8 @@ mod tests {
};
let algorithm: Arc<dyn Algorithm> = Arc::new(Passthrough::new(LlmTarget {
semantic_name: MODEL_ID.to_string(),
- llm_client: Some(Arc::new(EchoClient)),
}));
- let (trace, response) = algorithm.run(Context::default(), request).await?;
+ let (trace, response) = drive(algorithm, Context::default(), request, echo()).await?;
assert_eq!(
response
diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rs
index 3f41500c5..caa3fecc5 100644
--- a/crates/libsy/src/algorithms/rand.rs
+++ b/crates/libsy/src/algorithms/rand.rs
@@ -177,30 +177,13 @@ mod tests {
use super::*;
use std::collections::HashSet;
- use switchyard_protocol::{Metadata, completion_text, text_request, text_response};
+ use switchyard_protocol::{Metadata, completion_text, text_request};
use crate::DriverError;
use crate::algorithms::util::affinity::AffinityRouter;
use crate::core::algorithm::LlmTarget;
- use switchyard_protocol::{Decision, LlmResponse, Request, RoutedLlmClient, Signals};
-
- /// Echoes the selected target so tests can inspect which target was called.
- struct EchoClient;
-
- #[async_trait]
- impl RoutedLlmClient for EchoClient {
- async fn call(
- &self,
- _ctx: Context,
- _request: Request,
- decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(None, decision.selected_model())),
- metadata: None,
- })
- }
- }
+ use crate::core::testing::{drive, echo};
+ use switchyard_protocol::{Request, Signals};
fn request() -> Request {
Request {
@@ -225,13 +208,11 @@ mod tests {
.iter()
.map(|name| LlmTarget {
semantic_name: (*name).to_string(),
- llm_client: Some(Arc::new(EchoClient)),
})
.collect();
LlmTargetSet::new(targets)
}
- /// Builds a random router whose targets all share an echo client.
fn algorithm(names: &[&str], weights: Option<Vec<f64>>, seed: Option<u64>) -> Result<Random> {
Random::new(target_set(names), weights, seed)
}
@@ -243,7 +224,8 @@ mod tests {
async fn selected_models(algorithm: Arc<dyn Algorithm>, count: usize) -> Result<Vec<String>> {
let mut selected = Vec::with_capacity(count);
for _ in 0..count {
- let (_, response) = algorithm.clone().run(Context::default(), request()).await?;
+ let (_, response) =
+ drive(algorithm.clone(), Context::default(), request(), echo()).await?;
selected.push(
response
.llm_response
@@ -258,7 +240,7 @@ mod tests {
#[tokio::test]
async fn single_target_is_always_selected_and_called() -> Result<()> {
let algorithm = shared_algorithm(&["only/model"])?;
- let (trace, response) = algorithm.run(Context::default(), request()).await?;
+ let (trace, response) = drive(algorithm, Context::default(), request(), echo()).await?;
assert_eq!(
response
@@ -279,7 +261,8 @@ mod tests {
let algorithm = shared_algorithm(&names)?;
for _ in 0..50 {
- let (trace, response) = algorithm.clone().run(Context::default(), request()).await?;
+ let (trace, response) =
+ drive(algorithm.clone(), Context::default(), request(), echo()).await?;
let selected = response
.llm_response
.as_agg()
@@ -300,7 +283,8 @@ mod tests {
let mut seen = HashSet::new();
for _ in 0..100 {
- let (_, response) = algorithm.clone().run(Context::default(), request()).await?;
+ let (_, response) =
+ drive(algorithm.clone(), Context::default(), request(), echo()).await?;
seen.insert(
response
.llm_response
@@ -364,10 +348,13 @@ mod tests {
.with_classifier(random),
);
- let (_, first) = algorithm
- .clone()
- .run(Context::default(), request_for_session("session-1"))
- .await?;
+ let (_, first) = drive(
+ algorithm.clone(),
+ Context::default(),
+ request_for_session("session-1"),
+ echo(),
+ )
+ .await?;
let selected = first
.llm_response
.as_agg()
@@ -386,9 +373,13 @@ mod tests {
Some(selected.to_string())
);
- let (_, second) = algorithm
- .run(Context::default(), request_for_session("session-1"))
- .await?;
+ let (_, second) = drive(
+ algorithm,
+ Context::default(),
+ request_for_session("session-1"),
+ echo(),
+ )
+ .await?;
assert_eq!(
second
.llm_response
@@ -440,7 +431,7 @@ mod tests {
#[tokio::test]
async fn decision_is_inspectable_and_downcasts() -> Result<()> {
let algorithm = shared_algorithm(&["only/model"])?;
- let (trace, _) = algorithm.run(Context::default(), request()).await?;
+ let (trace, _) = drive(algorithm, Context::default(), request(), echo()).await?;
let decision = &trace[0];
assert_eq!(decision.selected_model(), "only/model");
diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs
index c82577c5b..b7a370e5a 100644
--- a/crates/libsy/src/algorithms/stage.rs
+++ b/crates/libsy/src/algorithms/stage.rs
@@ -223,22 +223,20 @@ mod tests {
use parking_lot::Mutex;
use serde_json::json;
use switchyard_protocol::{
- ContentBlock, LlmRequest, Message, Role, ToolCall, ToolResult, WireFormat, text_response,
+ ContentBlock, LlmRequest, Message, Role, ToolCall, ToolResult, WireFormat,
};
use super::*;
use crate::algorithms::util::stage::DECISION_SOURCE_KEY;
- use crate::core::algorithm::{Algorithm, LlmTarget};
+ use crate::core::algorithm::LlmTarget;
use crate::core::classifier::Score;
use crate::core::state::StateValue;
- use switchyard_protocol::{
- Context, Decision, LlmResponse, Metadata, Response, RoutedLlmClient,
- };
+ use crate::core::testing::{Serve, drive, reply};
+ use switchyard_protocol::{Context, Decision, Metadata, Response};
fn tier_target(name: &str) -> LlmTarget {
LlmTarget {
semantic_name: name.to_string(),
- llm_client: None,
}
}
@@ -329,7 +327,6 @@ mod tests {
config.llm_fallback = Some(LlmFallback {
judge_target: LlmTarget {
semantic_name: "judge".to_string(),
- llm_client: None,
},
config: TaskClassifierConfig {
base_threshold: -0.1,
@@ -360,16 +357,14 @@ mod tests {
messages: Vec<String>,
}
- /// Records what each target receives. When called as the judge target it
- /// replies with a structured verdict so the fallback classifier gets an answer
- /// without a real model.
+ /// Records what each target receives.
#[derive(Default)]
- struct RecordingClient {
+ struct Recorder {
calls: Mutex<Vec<Call>>,
judge_p_solve: Mutex<f64>,
}
- impl RecordingClient {
+ impl Recorder {
fn routed(&self) -> Vec<Call> {
self.calls
.lock()
@@ -378,55 +373,48 @@ mod tests {
.cloned()
.collect()
}
- }
- #[async_trait]
- impl RoutedLlmClient for RecordingClient {
- async fn call(
- &self,
- _ctx: Context,
- request: Request,
- decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {
- let target = decision.selected_model().to_string();
- self.calls.lock().push(Call {
- target: target.clone(),
- messages: request
- .llm_request
- .messages
- .iter()
- .filter_map(|message| message.text_content("|"))
- .collect(),
- });
- let completion = if target == JUDGE {
- let p_solve = *self.judge_p_solve.lock();
- format!(
- r#"{{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":{p_solve}}}"#
- )
- } else {
- target
- };
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(None, completion)),
- metadata: None,
- })
+ /// Serves every call, recording it. The judge target gets a structured verdict
+ /// back so the fallback classifier has an answer without a real model.
+ fn serve(self: &Arc<Self>) -> impl Serve {
+ let recorder = Arc::clone(self);
+ move |decision: Arc<dyn Decision>, request: Request| {
+ let recorder = Arc::clone(&recorder);
+ async move {
+ let target = decision.selected_model().to_string();
+ recorder.calls.lock().push(Call {
+ target: target.clone(),
+ messages: request
+ .llm_request
+ .messages
+ .iter()
+ .filter_map(|message| message.text_content("|"))
+ .collect(),
+ });
+ let completion = if target == JUDGE {
+ let p_solve = *recorder.judge_p_solve.lock();
+ format!(
+ r#"{{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":{p_solve}}}"#
+ )
+ } else {
+ target
+ };
+ Ok(reply(completion))
+ }
+ }
}
}
- fn recording_target(client: &Arc<RecordingClient>, name: &str) -> LlmTarget {
+ fn recording_target(name: &str) -> LlmTarget {
LlmTarget {
semantic_name: name.to_string(),
- llm_client: Some(client.clone() as Arc<dyn RoutedLlmClient>),
}
}
- fn recording_router(
- client: Arc<RecordingClient>,
- config: StageRouterConfig,
- ) -> Result<Arc<StageRouter>> {
+ fn recording_router(config: StageRouterConfig) -> Result<Arc<StageRouter>> {
Ok(Arc::new(StageRouter::new(
- recording_target(&client, "strong"),
- recording_target(&client, "weak"),
+ recording_target("strong"),
+ recording_target("weak"),
config,
)?))
}
@@ -437,11 +425,11 @@ mod tests {
c
}
- fn config_with_judge(client: &Arc<RecordingClient>, p_solve: f64) -> StageRouterConfig {
- *client.judge_p_solve.lock() = p_solve;
+ fn config_with_judge(recorder: &Arc<Recorder>, p_solve: f64) -> StageRouterConfig {
+ *recorder.judge_p_solve.lock() = p_solve;
let mut c = config();
c.llm_fallback = Some(LlmFallback {
- judge_target: recording_target(client, JUDGE),
+ judge_target: recording_target(JUDGE),
config: TaskClassifierConfig {
base_threshold: 0.5,
recent_turn_window: Some(3),
@@ -502,14 +490,20 @@ mod tests {
#[tokio::test]
async fn a_signal_driven_escalation_hands_the_note_to_the_model() -> Result<()> {
- let client = Arc::new(RecordingClient::default());
- let router = recording_router(client.clone(), config_with_notes())?;
+ let recorder = Arc::new(Recorder::default());
+ let router = recording_router(config_with_notes())?;
let ctx = Context::default();
- router.clone().run(ctx.clone(), turn_request(false)).await?;
- router.run(ctx, turn_request(true)).await?;
+ drive(
+ router.clone(),
+ ctx.clone(),
+ turn_request(false),
+ recorder.serve(),
+ )
+ .await?;
+ drive(router.clone(), ctx, turn_request(true), recorder.serve()).await?;
- let calls = client.routed();
+ let calls = recorder.routed();
assert_eq!(calls[0].target, "weak");
assert_eq!(calls[1].target, "strong");
assert!(
@@ -530,49 +524,67 @@ mod tests {
#[tokio::test]
async fn the_judge_decides_a_turn_the_signals_leave_undecided() -> Result<()> {
- let client = Arc::new(RecordingClient::default());
- let router = recording_router(client.clone(), config_with_judge(&client, 0.1))?;
+ let recorder = Arc::new(Recorder::default());
+ let router = recording_router(config_with_judge(&recorder, 0.1))?;
- router.run(Context::default(), turn_request(false)).await?;
+ drive(
+ router.clone(),
+ Context::default(),
+ turn_request(false),
+ recorder.serve(),
+ )
+ .await?;
assert!(
- client.calls.lock().iter().any(|c| c.target == JUDGE),
+ recorder.calls.lock().iter().any(|c| c.target == JUDGE),
"the judge should be consulted on an undecided turn"
);
- assert_eq!(client.routed()[0].target, "strong");
+ assert_eq!(recorder.routed()[0].target, "strong");
Ok(())
}
#[tokio::test]
async fn a_decisive_signal_never_reaches_the_judge() -> Result<()> {
- let client = Arc::new(RecordingClient::default());
- let router = recording_router(client.clone(), config_with_judge(&client, 0.9))?;
+ let recorder = Arc::new(Recorder::default());
+ let router = recording_router(config_with_judge(&recorder, 0.9))?;
- router.run(Context::default(), turn_request(true)).await?;
+ drive(
+ router.clone(),
+ Context::default(),
+ turn_request(true),
+ recorder.serve(),
+ )
+ .await?;
assert!(
- !client.calls.lock().iter().any(|c| c.target == JUDGE),
+ !recorder.calls.lock().iter().any(|c| c.target == JUDGE),
"a resolved turn should not pay for a judge call"
);
- assert_eq!(client.routed()[0].target, "strong");
+ assert_eq!(recorder.routed()[0].target, "strong");
Ok(())
}
#[tokio::test]
async fn the_judges_verdict_is_not_pinned_to_the_session() -> Result<()> {
- let client = Arc::new(RecordingClient::default());
- let router = recording_router(client.clone(), config_with_judge(&client, 0.1))?;
+ let recorder = Arc::new(Recorder::default());
+ let router = recording_router(config_with_judge(&recorder, 0.1))?;
let ctx = Context::default();
- router.clone().run(ctx.clone(), turn_request(false)).await?;
- *client.judge_p_solve.lock() = 0.9;
- router.run(ctx, turn_request(false)).await?;
-
- let routed = client.routed();
+ drive(
+ router.clone(),
+ ctx.clone(),
+ turn_request(false),
+ recorder.serve(),
+ )
+ .await?;
+ *recorder.judge_p_solve.lock() = 0.9;
+ drive(router.clone(), ctx, turn_request(false), recorder.serve()).await?;
+
+ let routed = recorder.routed();
assert_eq!(routed[0].target, "strong");
assert_eq!(routed[1].target, "weak");
assert_eq!(
- client
+ recorder
.calls
.lock()
.iter()
@@ -586,23 +598,35 @@ mod tests {
#[tokio::test]
async fn a_judge_that_cannot_tell_lands_on_the_picker_default() -> Result<()> {
- let client = Arc::new(RecordingClient::default());
- let router = recording_router(client.clone(), config_with_judge(&client, 42.0))?;
-
- router.run(Context::default(), turn_request(false)).await?;
-
- assert_eq!(client.routed()[0].target, "weak");
+ let recorder = Arc::new(Recorder::default());
+ let router = recording_router(config_with_judge(&recorder, 42.0))?;
+
+ drive(
+ router.clone(),
+ Context::default(),
+ turn_request(false),
+ recorder.serve(),
+ )
+ .await?;
+
+ assert_eq!(recorder.routed()[0].target, "weak");
Ok(())
}
#[tokio::test]
async fn the_judge_reads_the_window_it_was_configured_with() -> Result<()> {
- let client = Arc::new(RecordingClient::default());
- let router = recording_router(client.clone(), config_with_judge(&client, 0.9))?;
-
- router.run(Context::default(), turn_request(false)).await?;
-
- let judged = client
+ let recorder = Arc::new(Recorder::default());
+ let router = recording_router(config_with_judge(&recorder, 0.9))?;
+
+ drive(
+ router.clone(),
+ Context::default(),
+ turn_request(false),
+ recorder.serve(),
+ )
+ .await?;
+
+ let judged = recorder
.calls
.lock()
.iter()
diff --git a/crates/libsy/src/algorithms/subagent_affinity_tests.rs b/crates/libsy/src/algorithms/subagent_affinity_tests.rs
index 1e7198887..87ef0fcdc 100644
--- a/crates/libsy/src/algorithms/subagent_affinity_tests.rs
+++ b/crates/libsy/src/algorithms/subagent_affinity_tests.rs
@@ -15,31 +15,13 @@ use super::fall_through::FallThrough;
use super::util::affinity::AffinityRouter;
use super::util::subagent::SubagentOverride;
use crate::Result;
-use crate::core::algorithm::{Algorithm, Driver, LlmTarget, LlmTargetSet};
+use crate::core::algorithm::{Driver, LlmTarget, LlmTargetSet};
use crate::core::classifier::{Classification, Classifier, Score};
+use crate::core::testing::{drive, echo};
use switchyard_protocol::{
- Context, Decision, LlmResponse, Metadata, Request, Response, RoutedLlmClient, completion_text,
- slice_to_header_map, text_request, text_response,
+ Context, Metadata, Request, Response, completion_text, slice_to_header_map, text_request,
};
-/// A client that echoes the routed target name back as the completion.
-struct EchoClient;
-
-#[async_trait]
-impl RoutedLlmClient for EchoClient {
- async fn call(
- &self,
- _ctx: Context,
- _request: Request,
- decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(None, decision.selected_model())),
- metadata: None,
- })
- }
-}
-
/// The cascade's terminal classifier: always picks the orchestrator.
struct AlwaysOrchestrator;
@@ -67,7 +49,6 @@ fn targets() -> LlmTargetSet {
.iter()
.map(|name| LlmTarget {
semantic_name: (*name).to_string(),
- llm_client: Some(Arc::new(EchoClient) as Arc<dyn RoutedLlmClient>),
})
.collect(),
)
@@ -96,10 +77,7 @@ fn router() -> Arc<FallThrough> {
/// Runs one turn, returning the target that served it.
async fn turn(router: &Arc<FallThrough>, headers: &[(&str, &str)]) -> Result<String> {
- let (_, response) = router
- .clone()
- .run(Context::default(), request(headers))
- .await?;
+ let (_, response) = drive(router.clone(), Context::default(), request(headers), echo()).await?;
Ok(response
.llm_response
.as_agg()
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
index 3898c44fb..2f281187e 100644
--- a/crates/libsy/src/algorithms/util/llm_judge.rs
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -18,7 +18,7 @@ use switchyard_protocol::{
};
use super::classifier_contract::ClassifierContract;
-use crate::core::algorithm::{Driver, LlmTarget};
+use crate::core::algorithm::{Driver, LlmTarget, RoutedRequest};
use crate::core::classifier::{Classification, Classifier};
use crate::core::state::State;
use crate::{LibsyError, Result};
@@ -227,14 +227,13 @@ where
let judge_model = self.target.semantic_name.as_str();
let response = driver
- .call_llm_target(
- Context::default(),
- &self.target,
- self.judge.build_request(state, request),
- Arc::new(JudgeDecision {
+ .call_llm(RoutedRequest {
+ request: self.judge.build_request(state, request),
+ decision: Arc::new(JudgeDecision {
model: self.target.semantic_name.to_string(),
}),
- )
+ ctx: Context::default(),
+ })
.await
.inspect_err(|error| report_fail_open(judge_model, error, libsy_error_reason(error)))
.ok()?;
@@ -409,7 +408,6 @@ mod tests {
TestJudge,
LlmTarget {
semantic_name: "judge".to_string(),
- llm_client: None,
},
TestPolicy,
)
diff --git a/crates/libsy/src/core.rs b/crates/libsy/src/core.rs
index b5655269e..e922faf13 100644
--- a/crates/libsy/src/core.rs
+++ b/crates/libsy/src/core.rs
@@ -6,6 +6,8 @@
//! pump in [`driver`]. Algorithm implementations live in [`crate::algorithms`].
mod driver;
+#[cfg(test)]
+pub(crate) mod testing;
pub mod algorithm;
pub mod classifier;
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index 0d4e76f9f..d7179eb24 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -7,6 +7,7 @@
use std::{
collections::{HashMap, HashSet},
+ future::Future,
pin::Pin,
sync::Arc,
time::{Duration, Instant},
@@ -25,8 +26,7 @@ use tracing::Instrument;
/// [`switchyard_protocol::LlmResponse`] carries either a live
/// [`switchyard_protocol::LlmResponseStream`] or the terminal aggregate.
use switchyard_protocol::{
- Context, Decision, LlmClientError, Request, Response, RoutedLlmClient, RoutingFallbackReason,
- Signals, Usage,
+ Context, Decision, LlmClientError, Request, Response, RoutingFallbackReason, Signals, Usage,
};
use super::driver::{DriverRequest, DriverStep, TypeErasedDriver};
@@ -84,13 +84,8 @@ pub struct RoutedRequest {
pub request: Request,
/// The routing decision behind this call; `selected_model()` is the model to hit.
pub decision: Arc<dyn Decision>,
- /// The client that serves this call by default, or `None` when the routed target
- /// had no client. Rides along on the offloaded call so a host driving the stream
- /// can serve it by default or override it with its own transport.
- pub default_client: Option<Arc<dyn RoutedLlmClient>>,
- /// The request's cross-cutting context, carried through the offload so whoever
- /// serves the call (libsy's own `run`, or a host driving the stream) hands it to
- /// [`RoutedLlmClient::call`].
+ /// The request's cross-cutting context, carried through the offload so the host
+ /// serving the call can pass it to its client.
pub ctx: Context,
}
@@ -100,7 +95,8 @@ pub struct RoutedRequest {
/// routed request ([`get_routed`](Self::get_routed)) and the decision behind it
/// ([`get_decision`](Self::get_decision)), performs (or delegates) the model call, and
/// fulfills it with [`respond`](Self::respond) — unblocking the algorithm's
-/// [`Driver::call_llm`] on the other side.
+/// [`Driver::call_llm`] on the other side. `switchyard-llm-client`'s `run` is the
+/// ready-made consumer that does this for you.
pub struct CallLlmRequest {
inner: DriverRequest,
routed: RoutedRequest,
@@ -119,9 +115,8 @@ impl CallLlmRequest {
Self { inner, routed }
}
- /// The routed request the host should serve. Its
- /// [`default_client`](RoutedRequest::default_client) serves the call by default,
- /// and its `decision.selected_model()` names the model to hit.
+ /// The routed request the host should serve; its `decision.selected_model()` names
+ /// the model to hit.
pub fn get_routed(&self) -> &RoutedRequest {
&self.routed
}
@@ -146,10 +141,10 @@ impl CallLlmRequest {
/// The offload channel handed to an algorithm's
/// [`create_run_task`](Algorithm::create_run_task). The algorithm makes model calls
-/// with [`call_llm_target`](Self::call_llm_target) (or [`call_llm`](Self::call_llm)) and
-/// publishes its [`Decision`]s with [`info`](Self::info); each call is offloaded to the
-/// request's [`Step`] stream and awaits the consumer's response. The step channel is
-/// bounded, so the consumer paces the algorithm one step at a time.
+/// with [`call_llm`](Self::call_llm) and publishes its [`Decision`]s with
+/// [`info`](Self::info); each call is offloaded to the request's [`Step`] stream and
+/// awaits the consumer's response. The step channel is bounded, so the consumer paces
+/// the algorithm one step at a time.
#[derive(Clone)]
pub struct Driver {
driver: TypeErasedDriver,
@@ -192,8 +187,8 @@ impl Driver {
/// The await is wrapped in a `libsy.llm_call` span measuring *fulfillment* as
/// the algorithm observes it (host queueing/serving included; a streamed
/// response resolves when its stream handle arrives); latency, outcome, and
- /// token usage are recorded when it resolves. The provider call itself gets a
- /// `libsy.client_call` span when [`Algorithm::run`] serves it.
+ /// token usage are recorded when it resolves. The provider call itself is the
+ /// host's, and is instrumented by whoever makes it.
#[tracing::instrument(
target = "libsy",
name = "libsy.llm_call",
@@ -252,27 +247,6 @@ impl Driver {
result
}
- /// Offload a call to `target`: pair `request` with `decision` and the target's
- /// default client into a [`RoutedRequest`], then publish it (see
- /// [`call_llm`](Self::call_llm)). The convenience most algorithms use;
- /// `decision.selected_model()` names the model to hit, and `request`'s
- /// `model` is left untouched.
- pub async fn call_llm_target(
- &self,
- ctx: Context,
- target: &LlmTarget,
- request: Request,
- decision: Arc<dyn Decision>,
- ) -> Result<Response> {
- self.call_llm(RoutedRequest {
- request,
- decision,
- default_client: target.llm_client.clone(),
- ctx,
- })
- .await
- }
-
/// Publish a routing [`Decision`] as a [`Step::Decision`] on the stream.
/// Each successfully published decision is counted and logged with its
/// reasoning; a decision the stream never accepted is not recorded.
@@ -328,9 +302,8 @@ impl Default for Driver {
/// One item in the stream returned by `Driver::stream` / [`Algorithm::run_stream`].
pub enum Step {
- /// The algorithm needs this model call performed. The host serves it (optionally
- /// via [`RoutedRequest::default_client`]) and fulfills it with
- /// [`CallLlmRequest::respond`]. Boxed: it is by far the largest variant.
+ /// The algorithm needs this model call performed. The host serves it and fulfills
+ /// it with [`CallLlmRequest::respond`]. Boxed: it is by far the largest variant.
CallLlm(Box<CallLlmRequest>),
/// A routing decision the algorithm made, published via [`Driver::info`] as it
/// happens (rather than collected into a trace returned at the end).
@@ -339,6 +312,64 @@ pub enum Step {
ReturnToAgent(Box<Response>),
}
+/// Drive [`Algorithm::run_stream`] to completion, handing each offloaded call to `serve`.
+///
+/// Returns the final [`Response`] and the trace of [`Decision`]s the algorithm published.
+/// `observer`, when present, is passed through to `run_stream`.
+///
+/// `serve` owns the call: it performs it however the host likes and must fulfill the promise
+/// with [`CallLlmRequest::respond`]. A failed *model* call belongs in `respond` — the
+/// algorithm may route around it. Returning `Err` from `serve` aborts the whole run, so
+/// reserve it for infrastructure failures. Calls are served concurrently, so an algorithm
+/// that offloads several at once (hedging, fan-out) gets real parallelism.
+///
+/// libsy performs no I/O; this is only the mechanics of consuming its own step stream, kept
+/// here so every host does not reimplement the same loop. `switchyard-llm-client`'s `run`
+/// is this function plus an HTTP client.
+pub async fn drive<F, Fut>(
+ algorithm: Arc<dyn Algorithm>,
+ ctx: Context,
+ request: Request,
+ observer: Option<RunObserver>,
+ serve: F,
+) -> Result<(Vec<Arc<dyn Decision>>, Response)>
+where
+ F: Fn(CallLlmRequest) -> Fut,
+ Fut: Future<Output = Result<()>>,
+{
+ let stream = algorithm.run_stream(ctx, request, observer);
+ tokio::pin!(stream);
+
+ let mut trace: Vec<Arc<dyn Decision>> = Vec::new();
+ let mut in_flight = futures::stream::FuturesUnordered::new();
+ let mut final_response: Option<Response> = None;
+
+ loop {
+ tokio::select! {
+ Some(result) = in_flight.next() => match result {
+ Ok(()) => {}, // CallLlm completed successfully
+ Err(err) => return Err(err), // CallLlm failed, propagate the error
+ },
+ step = stream.next() => {
+ match step {
+ None => break, // stream has ended, no more steps
+ Some(item) => match item? {
+ Step::CallLlm(call) => in_flight.push(serve(*call)),
+ Step::Decision(decision) => trace.push(decision),
+ Step::ReturnToAgent(response) => {
+ final_response = Some(*response);
+ break;
+ }
+ }
+ }
+ },
+ }
+ }
+ final_response
+ .map(|response| (trace, response))
+ .ok_or(LibsyError::MissingFinalResponse)
+}
+
/// Abort guard
struct AbortOnDrop(tokio::task::AbortHandle);
@@ -348,19 +379,15 @@ impl Drop for AbortOnDrop {
}
}
-/// A named routing target: a `semantic_name` an algorithm routes by, and an optional
-/// [`RoutedLlmClient`] to serve its calls. An algorithm hands a target to
-/// [`Driver::call_llm_target`]; the client rides along as
-/// [`RoutedRequest::default_client`] for the stream consumer to serve or override.
+/// A named routing target an algorithm routes by. Serving its calls is the stream
+/// consumer's concern: the target's name reaches the consumer as
+/// `decision.selected_model()` on the offloaded [`RoutedRequest`].
#[derive(Clone)]
pub struct LlmTarget {
/// The routing label an algorithm selects this target by — a logical tier like
/// `"strong"`, or the model id when they coincide. Mapping it to a provider model
- /// id is the client's concern, never the algorithm's.
+ /// id is the consumer's concern, never the algorithm's.
pub semantic_name: String,
- /// The client that serves this target's calls by default, or `None` (then the
- /// stream consumer must serve them).
- pub llm_client: Option<Arc<dyn RoutedLlmClient>>,
}
/// The set of targets an algorithm may route among. An algorithm is constructed
@@ -567,7 +594,11 @@ pub(crate) async fn call_llm_with_fallback(
) -> Result<Response> {
loop {
let result = driver
- .call_llm_target(ctx.clone(), &target, request.clone(), decision.clone())
+ .call_llm(RoutedRequest {
+ request: request.clone(),
+ decision: decision.clone(),
+ ctx: ctx.clone(),
+ })
.await;
let Err(error) = result else { return result };
let Some((failed, reason)) = classify_fallback(&error) else {
@@ -592,8 +623,9 @@ pub(crate) async fn call_llm_with_fallback(
}
/// An optimization strategy. Implement [`create_run_task`](Self::create_run_task);
-/// callers drive it with the provided [`run`](Self::run) (serve calls, get the answer)
-/// or [`run_stream`](Self::run_stream) (drive the [`Step`] stream yourself).
+/// callers drive it with [`run_stream`](Self::run_stream), serving each [`Step::CallLlm`]
+/// it emits. `switchyard-llm-client`'s `run` is the ready-made consumer that does this
+/// over HTTP.
///
/// Methods take `self: Arc<Self>`: one algorithm (`Arc<dyn Algorithm>`) is shared across
/// requests and run concurrently, so it owns its thread-safety and any shared state.
@@ -608,9 +640,9 @@ pub(crate) async fn call_llm_with_fallback(
/// # Observability
///
/// [`run_stream`](Self::run_stream) creates a `libsy.run` span, and each offloaded model
-/// call creates a `libsy.llm_call` span. [`run`](Self::run) additionally wraps calls it
-/// serves through a target's default client in `libsy.client_call`. Decisions and failures
-/// are emitted through `tracing`; metrics use the global OpenTelemetry meter provider.
+/// call creates a `libsy.llm_call` span. Decisions and failures are emitted through
+/// `tracing`; metrics use the global OpenTelemetry meter provider. The provider call
+/// itself belongs to the host, and is instrumented by whoever makes it.
#[async_trait]
pub trait Algorithm: Send + Sync + 'static {
/// Stable, low-cardinality name identifying this algorithm — the
@@ -618,11 +650,11 @@ pub trait Algorithm: Send + Sync + 'static {
/// emits for its runs.
fn name(&self) -> &str;
- /// Run one request to completion: make model calls with [`Driver::call_llm_target`],
+ /// Run one request to completion: make model calls with [`Driver::call_llm`],
/// publish [`Decision`]s with [`Driver::info`], and return the final [`Response`].
- /// The method an algorithm implements; [`run`](Self::run) / [`run_stream`](Self::run_stream)
- /// drive it. `ctx` carries the request's cross-cutting values (today: the
- /// algorithm's telemetry label in [`Context::values`]).
+ /// The method an algorithm implements; [`run_stream`](Self::run_stream) drives it.
+ /// `ctx` carries the request's cross-cutting values (today: the algorithm's
+ /// telemetry label in [`Context::values`]).
async fn create_run_task(
self: Arc<Self>,
ctx: Context,
@@ -703,136 +735,12 @@ pub trait Algorithm: Send + Sync + 'static {
step
}))
}
-
- /// Process a request to completion, returning the final [`Response`] and the trace of
- /// [`Decision`]s the algorithm made along the way.
- ///
- async fn run(
- self: Arc<Self>,
- ctx: Context,
- request: Request,
- ) -> Result<(Vec<Arc<dyn Decision>>, Response)> {
- self.run_observed(ctx, request, None).await
- }
-
- /// Process a request to completion while reporting each model call to `observer`.
- async fn run_observed(
- self: Arc<Self>,
- ctx: Context,
- request: Request,
- observer: Option<RunObserver>,
- ) -> Result<(Vec<Arc<dyn Decision>>, Response)> {
- // Serve one offloaded call with its target's default client. A failed *model*
- // call is forwarded to the algorithm via `respond`; this errors only on an
- // infrastructure failure (no default client, or the promise was dropped).
- // `serve` makes the one API call libsy itself performs, so it gets its
- // own `libsy.client_call` span.
- #[tracing::instrument(
- target = "libsy",
- name = "libsy.client_call",
- skip_all,
- fields(
- algorithm = observability::algorithm_label(&call.get_routed().ctx),
- switchyard.algorithm = observability::algorithm_label(&call.get_routed().ctx),
- switchyard.routing.tier = tracing::field::Empty,
- selected_model = call.get_decision().selected_model(),
- otel.kind = "client",
- otel.name = %format_args!("chat {}", call.get_decision().selected_model()),
- openinference.span.kind = "LLM",
- gen_ai.operation.name = "chat",
- gen_ai.request.model = call.get_decision().selected_model(),
- gen_ai.request.stream = tracing::field::Empty,
- gen_ai.request.temperature = tracing::field::Empty,
- gen_ai.request.top_p = tracing::field::Empty,
- gen_ai.request.top_k = tracing::field::Empty,
- gen_ai.request.max_tokens = tracing::field::Empty,
- gen_ai.request.reasoning.level = tracing::field::Empty,
- gen_ai.output.type = tracing::field::Empty,
- gen_ai.conversation.id = tracing::field::Empty,
- server.address = tracing::field::Empty,
- server.port = tracing::field::Empty,
- gen_ai.response.id = tracing::field::Empty,
- gen_ai.response.model = tracing::field::Empty,
- gen_ai.usage.input_tokens = tracing::field::Empty,
- gen_ai.usage.output_tokens = tracing::field::Empty,
- gen_ai.usage.cache_read.input_tokens = tracing::field::Empty,
- gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty,
- gen_ai.usage.reasoning.output_tokens = tracing::field::Empty,
- outcome = tracing::field::Empty,
- otel.status_code = tracing::field::Empty,
- error.type = tracing::field::Empty,
- error = tracing::field::Empty,
- )
- )]
- async fn serve(call: CallLlmRequest) -> Result<()> {
- let span = tracing::Span::current();
- observability::record_gen_ai_request(&span, &call.get_routed().request.llm_request);
- if let Some(tier) = call.get_decision().routing_tier() {
- span.record("switchyard.routing.tier", tier);
- }
- if let Some(session_id) = call
- .get_routed()
- .request
- .metadata
- .as_ref()
- .and_then(|metadata| metadata.session_id.as_deref())
- {
- span.record("gen_ai.conversation.id", session_id);
- }
- let routed = call.get_routed().clone();
- let target = routed.decision.selected_model().to_string();
- let client =
- routed
- .default_client
- .clone()
- .ok_or_else(|| LibsyError::MissingClient {
- target: target.clone(),
- })?;
- let result = client
- .call(routed.ctx, routed.request, routed.decision)
- .await
- .map_err(|source| LibsyError::client_call(target, source));
- let result = observability::observe_client_call(result);
- call.respond(result)
- }
-
- let stream = self.run_stream(ctx, request, observer);
- tokio::pin!(stream);
-
- let mut trace: Vec<Arc<dyn Decision>> = Vec::new();
- let mut in_flight = futures::stream::FuturesUnordered::new();
- let mut final_response: Option<Response> = None;
-
- loop {
- tokio::select! {
- Some(result) = in_flight.next() => match result {
- Ok(()) => {}, // CallLlm completed successfully
- Err(err) => return Err(err), // CallLlm failed, propagate the error
- },
- step = stream.next() => {
- match step {
- None => break, // stream has ended, no more steps
- Some(item) => match item? {
- Step::CallLlm(call) => in_flight.push(serve(*call)),
- Step::Decision(decision) => trace.push(decision),
- Step::ReturnToAgent(response) => {
- final_response = Some(*response);
- break;
- }
- }
- }
- },
- }
- }
- final_response
- .map(|response| (trace, response))
- .ok_or(LibsyError::MissingFinalResponse)
- }
}
#[cfg(test)]
mod tests {
use super::*;
+ use crate::core::testing::{Serve, ServeResult, drive, drive_observed, echo, reply};
use futures::StreamExt;
use switchyard_protocol::{
LlmResponse, LlmResponseChunk, completion_text, text_request, text_response,
@@ -901,28 +809,6 @@ mod tests {
);
}
- /// Mock client that echoes back the target name it was called with.
- struct EchoClient;
-
- #[async_trait]
- impl RoutedLlmClient for EchoClient {
- async fn call(
- &self,
- _ctx: Context,
- _request: Request,
- decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, LlmClientError> {
- // Echo back the model the algorithm routed to (the decision's selection).
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(
- None,
- decision.selected_model().to_string(),
- )),
- metadata: None,
- })
- }
- }
-
/// Trivial decision + algo used only to exercise the orchestrator: calls the
/// first target and returns its response with a one-item trace.
struct TestDecision {
@@ -968,7 +854,11 @@ mod tests {
});
driver.info(ctx.clone(), decision.clone()).await?;
driver
- .call_llm_target(ctx, &target, request, decision)
+ .call_llm(RoutedRequest {
+ request,
+ decision,
+ ctx,
+ })
.await
}
}
@@ -986,13 +876,11 @@ mod tests {
}
}
- /// `(name, has_client)` — `has_client: false` builds a target with no default client.
- fn target_set(names: &[(&str, bool)]) -> LlmTargetSet {
+ fn target_set(names: &[&str]) -> LlmTargetSet {
let targets = names
.iter()
- .map(|(name, has_client)| LlmTarget {
+ .map(|name| LlmTarget {
semantic_name: name.to_string(),
- llm_client: has_client.then(|| Arc::new(EchoClient) as Arc<dyn RoutedLlmClient>),
})
.collect();
LlmTargetSet::new(targets)
@@ -1003,9 +891,14 @@ mod tests {
let observations = Arc::new(Mutex::new(Vec::new()));
let observed = observations.clone();
let observer: RunObserver = Arc::new(move |observation| observed.lock().push(observation));
- let (_, response) = orch(target_set(&[("direct/model", true)]))
- .run_observed(Context::default(), request(), Some(observer))
- .await?;
+ let (_, response) = drive_observed(
+ orch(target_set(&["direct/model"])),
+ Context::default(),
+ request(),
+ Some(observer),
+ echo(),
+ )
+ .await?;
assert_eq!(
response.llm_response.as_agg().map(completion_text),
Some("direct/model".to_string())
@@ -1035,48 +928,29 @@ mod tests {
));
}
- /// Client that serves a call as a token stream — its `call` returns
- /// [`LlmResponse::Stream`] replaying `chunks` in order (as `Ok` items).
- struct StreamingClient {
- chunks: Vec<LlmResponseChunk>,
- }
-
- #[async_trait]
- impl RoutedLlmClient for StreamingClient {
- async fn call(
- &self,
- _ctx: Context,
- _request: Request,
- _decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, LlmClientError> {
- let stream = futures::stream::iter(
- self.chunks
- .clone()
- .into_iter()
- .map(|chunk| Ok(chunk.into())),
- )
- .boxed();
- Ok(Response {
- llm_response: LlmResponse::Stream(stream),
- metadata: None,
- })
- }
- }
-
- /// Build a single-target algo whose one target streams `chunks`.
- fn streaming_orch(chunks: Vec<LlmResponseChunk>) -> Arc<dyn Algorithm> {
- let target = LlmTarget {
- semantic_name: "stream/model".to_string(),
- llm_client: Some(Arc::new(StreamingClient { chunks }) as Arc<dyn RoutedLlmClient>),
+ /// Build a single-target algo, plus a `serve` that answers it as a token stream
+ /// replaying `chunks` in order (as `Ok` items).
+ fn streaming_orch(chunks: Vec<LlmResponseChunk>) -> (Arc<dyn Algorithm>, impl Serve) {
+ let algo = orch(target_set(&["stream/model"]));
+ let serve = move |_decision: Arc<dyn Decision>, _request: Request| {
+ let chunks = chunks.clone();
+ async move {
+ let stream =
+ futures::stream::iter(chunks.into_iter().map(|chunk| Ok(chunk.into()))).boxed();
+ Ok(Response {
+ llm_response: LlmResponse::Stream(stream),
+ metadata: None,
+ })
+ }
};
- orch(LlmTargetSet::new(vec![target]))
+ (algo, serve)
}
#[tokio::test]
async fn run_returns_a_streamed_response_the_caller_aggregates() -> Result<()> {
// A streaming client -> its chunks flow through the promise and `ReturnToAgent`,
// and `run` returns the live stream untouched for the caller to fold.
- let orch = streaming_orch(vec![
+ let (orch, serve) = streaming_orch(vec![
LlmResponseChunk::MessageStart {
id: Some("m1".to_string()),
model: Some("stream/model".to_string()),
@@ -1093,8 +967,8 @@ mod tests {
reason: Some("stop".to_string()),
},
]);
- let (trace, response) = orch.run(Context::default(), request()).await?;
- // `run` handed back the live stream; the caller folds it to a buffered aggregate.
+ let (trace, response) = drive(orch, Context::default(), request(), serve).await?;
+ // The run handed back the live stream; the caller folds it to a buffered aggregate.
let agg = response
.llm_response
.into_agg()
@@ -1108,9 +982,9 @@ mod tests {
#[tokio::test]
async fn aggregating_a_streamed_response_propagates_a_mid_stream_error() -> Result<()> {
- // `run` succeeds and returns the stream; the in-band `Error` chunk surfaces only
+ // The run succeeds and returns the stream; the in-band `Error` chunk surfaces only
// when the caller aggregates it.
- let orch = streaming_orch(vec![
+ let (orch, serve) = streaming_orch(vec![
LlmResponseChunk::TextDelta {
index: 0,
text: "partial".to_string(),
@@ -1119,7 +993,7 @@ mod tests {
message: "upstream exploded".to_string(),
},
]);
- let (_, response) = orch.run(Context::default(), request()).await?;
+ let (_, response) = drive(orch, Context::default(), request(), serve).await?;
match response.llm_response.into_agg().await {
Ok(_) => panic!("expected a mid-stream error, got an aggregate"),
Err(err) => {
@@ -1131,13 +1005,10 @@ mod tests {
#[tokio::test]
async fn run_offloads_via_promise_then_returns_to_agent() -> Result<()> {
- // A client-less target -> its call is offloaded via a promise the
- // orchestrator surfaces as a `CallLlm` step for us to fulfill.
- let stream = orch(target_set(&[("offload/model", false)])).run_stream(
- Context::default(),
- request(),
- None,
- );
+ // Every call is offloaded via a promise the orchestrator surfaces as a
+ // `CallLlm` step for us to fulfill.
+ let stream =
+ orch(target_set(&["offload/model"])).run_stream(Context::default(), request(), None);
tokio::pin!(stream);
let mut saw_call = false;
@@ -1181,61 +1052,15 @@ mod tests {
}
#[tokio::test]
- async fn client_backed_target_offloads_with_a_default_client() -> Result<()> {
- // Every call now offloads to the stream; a client-backed target rides its
- // client along as `default_client` so the consumer can serve it by default.
- let stream = orch(target_set(&[("direct/model", true)])).run_stream(
+ async fn a_driven_run_returns_the_trace_and_the_final_response() -> Result<()> {
+ let (trace, response) = drive(
+ orch(target_set(&["direct/model"])),
Context::default(),
request(),
- None,
- );
- tokio::pin!(stream);
-
- let mut final_completion = None;
- while let Some(step) = stream.next().await {
- match step? {
- Step::CallLlm(call) => {
- let routed = call.get_routed().clone();
- let client = routed
- .default_client
- .clone()
- .ok_or_else(|| test_error("expected a default client"))?;
- let target = routed.decision.selected_model().to_string();
- let result = client
- .call(routed.ctx, routed.request, routed.decision)
- .await
- .map_err(|error| LibsyError::client_call(target, error));
- call.respond(result)?;
- }
- Step::Decision(_) => {}
- Step::ReturnToAgent(response) => {
- final_completion = Some(
- response
- .llm_response
- .as_agg()
- .map(completion_text)
- .unwrap_or_default(),
- );
- }
- }
- }
-
- // EchoClient echoes the model name back as the completion.
- assert_eq!(
- final_completion.ok_or_else(|| test_error("no ReturnToAgent"))?,
- "direct/model"
- );
- Ok(())
- }
-
- #[tokio::test]
- async fn run_returns_the_response_when_all_targets_have_clients() -> Result<()> {
- // Every target has a client, so run serves every call via the
- // default client and returns the trace + final response.
- let (trace, response) = orch(target_set(&[("direct/model", true)]))
- .run(Context::default(), request())
- .await?;
- // TestAlgo calls the first target; EchoClient echoes its name.
+ echo(),
+ )
+ .await?;
+ // TestAlgo calls the first target; `echo` answers with its name.
assert_eq!(
response
.llm_response
@@ -1248,22 +1073,6 @@ mod tests {
Ok(())
}
- #[tokio::test]
- async fn run_errors_when_a_target_lacks_a_client() -> Result<()> {
- // A client-less target has no default client to serve its offloaded call, so
- // driving it to completion errors.
- let error = orch(target_set(&[("offload/model", false)]))
- .run(Context::default(), request())
- .await
- .err()
- .ok_or_else(|| test_error("expected a missing-client error"))?;
- assert!(matches!(
- error,
- LibsyError::MissingClient { target } if target == "offload/model"
- ));
- Ok(())
- }
-
#[tokio::test(flavor = "multi_thread", worker_threads = 12)]
async fn requests_are_processed_in_parallel() -> Result<()> {
use std::time::Duration;
@@ -1271,49 +1080,27 @@ mod tests {
const N: usize = 12;
- // A client that blocks until all N concurrent calls have arrived. If
- // requests were serialized (one algorithm behind a `Mutex`), only one
- // call could be in flight, the barrier would never reach N, and the test
- // would time out. It passes only because the shared algorithm is driven
- // concurrently across requests.
- struct BarrierClient {
- barrier: Arc<Barrier>,
- }
-
- #[async_trait]
- impl RoutedLlmClient for BarrierClient {
- async fn call(
- &self,
- _ctx: Context,
- _request: Request,
- decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, LlmClientError> {
- self.barrier.wait().await;
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(
- None,
- decision.selected_model().to_string(),
- )),
- metadata: None,
- })
- }
- }
-
+ // Serving blocks until all N concurrent calls have arrived. If requests were
+ // serialized (one algorithm behind a `Mutex`), only one call could be in flight,
+ // the barrier would never reach N, and the test would time out. It passes only
+ // because the shared algorithm is driven concurrently across requests.
let barrier = Arc::new(Barrier::new(N));
- let targets = LlmTargetSet::new(vec![LlmTarget {
- semantic_name: "m".to_string(),
- llm_client: Some(Arc::new(BarrierClient {
- barrier: barrier.clone(),
- })),
- }]);
// One shared algorithm driven by many concurrent requests.
- let algo = orch(targets);
+ let algo = orch(target_set(&["m"]));
let mut handles = Vec::new();
for _ in 0..N {
let algo = algo.clone();
+ let barrier = barrier.clone();
+ let serve = move |decision: Arc<dyn Decision>, _request: Request| {
+ let barrier = barrier.clone();
+ async move {
+ barrier.wait().await;
+ Ok(reply(decision.selected_model()))
+ }
+ };
handles.push(tokio::spawn(async move {
- algo.run(Context::default(), request())
+ drive(algo, Context::default(), request(), serve)
.await
.map(|(_, response)| {
response
@@ -1341,11 +1128,8 @@ mod tests {
// A client-less target offloads its call; we fulfill the promise with an
// Err, which must flow back through `call_llm_target` into the algorithm and
// out as an error step — not a response.
- let stream = orch(target_set(&[("offload/model", false)])).run_stream(
- Context::default(),
- request(),
- None,
- );
+ let stream =
+ orch(target_set(&["offload/model"])).run_stream(Context::default(), request(), None);
tokio::pin!(stream);
let mut saw_error = false;
@@ -1498,9 +1282,9 @@ mod tests {
}
let algo: Arc<dyn Algorithm> = Arc::new(Panicky);
- match algo.run(Context::default(), request()).await {
+ match drive(algo, Context::default(), request(), echo()).await {
Ok(_) => Err(test_error(
- "expected run to surface the algorithm panic as an error",
+ "expected the run to surface the algorithm panic as an error",
)),
Err(err) => {
assert!(matches!(err, LibsyError::AlgorithmTask { .. }));
@@ -1557,9 +1341,10 @@ mod tests {
dropped: dropped.clone(),
});
- // Drive `run` on its own task, wait until the algorithm task is up, then cancel
- // `run` — dropping its future (and the `run_stream` stream it holds).
- let run_task = tokio::spawn(async move { algo.run(Context::default(), request()).await });
+ // Drive the run on its own task, wait until the algorithm task is up, then cancel
+ // it — dropping its future (and the `run_stream` stream it holds).
+ let run_task =
+ tokio::spawn(async move { drive(algo, Context::default(), request(), echo()).await });
started_rx
.recv()
.await
@@ -1576,62 +1361,6 @@ mod tests {
// --- first-wins hedging: `run` must not wait on losing speculative calls -------------
- /// The loser: signals it has started serving (so the winner can then win with the
- /// loser's serve guaranteed in flight), then finishes late (`Some(delay)`) or never
- /// (`None`).
- struct LoserClient {
- started: Arc<tokio::sync::Notify>,
- delay: Option<std::time::Duration>,
- }
-
- #[async_trait]
- impl RoutedLlmClient for LoserClient {
- async fn call(
- &self,
- _ctx: Context,
- _request: Request,
- decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, LlmClientError> {
- self.started.notify_one();
- match self.delay {
- Some(delay) => tokio::time::sleep(delay).await,
- None => std::future::pending::<()>().await,
- }
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(
- None,
- decision.selected_model().to_string(),
- )),
- metadata: None,
- })
- }
- }
-
- /// The winner: waits until the loser's serve has started, then echoes immediately, so
- /// the loser's serve is guaranteed in flight when the winner wins.
- struct GatedEchoClient {
- gate: Arc<tokio::sync::Notify>,
- }
-
- #[async_trait]
- impl RoutedLlmClient for GatedEchoClient {
- async fn call(
- &self,
- _ctx: Context,
- _request: Request,
- decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, LlmClientError> {
- self.gate.notified().await;
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(
- None,
- decision.selected_model().to_string(),
- )),
- metadata: None,
- })
- }
- }
-
/// Offloads two targets concurrently and returns the first to resolve, dropping the
/// loser's call (first-wins hedging).
struct Hedge {
@@ -1657,8 +1386,16 @@ mod tests {
let dec_l: Arc<dyn Decision> = Arc::new(TestDecision {
model: self.loser.semantic_name.clone(),
});
- let win = driver.call_llm_target(ctx.clone(), &self.winner, request.clone(), dec_w);
- let lose = driver.call_llm_target(ctx, &self.loser, request, dec_l);
+ let win = driver.call_llm(RoutedRequest {
+ request: request.clone(),
+ decision: dec_w,
+ ctx: ctx.clone(),
+ });
+ let lose = driver.call_llm(RoutedRequest {
+ request,
+ decision: dec_l,
+ ctx,
+ });
// First to resolve wins; `select!` drops the losing future (and its promise).
tokio::select! {
res = win => res,
@@ -1667,33 +1404,43 @@ mod tests {
}
}
- /// Builds a hedging algo whose winner is gated behind the loser starting, and whose
- /// loser finishes after `loser_delay` (or never, when `None`).
- fn hedge(loser_delay: Option<std::time::Duration>) -> Arc<dyn Algorithm> {
+ /// Builds a hedging algo and the `serve` that drives it: the winner is gated behind the
+ /// loser starting (so the loser's serve is guaranteed in flight when the winner wins),
+ /// and the loser finishes after `loser_delay` — or never, when `None`.
+ fn hedge(loser_delay: Option<std::time::Duration>) -> (Arc<dyn Algorithm>, impl Serve) {
let started = Arc::new(tokio::sync::Notify::new());
- let winner = LlmTarget {
- semantic_name: "winner".to_string(),
- llm_client: Some(Arc::new(GatedEchoClient {
- gate: started.clone(),
- })),
- };
- let loser = LlmTarget {
- semantic_name: "loser".to_string(),
- llm_client: Some(Arc::new(LoserClient {
- started,
- delay: loser_delay,
- })),
+ let algo = Arc::new(Hedge {
+ winner: LlmTarget {
+ semantic_name: "winner".to_string(),
+ },
+ loser: LlmTarget {
+ semantic_name: "loser".to_string(),
+ },
+ });
+ let serve = move |decision: Arc<dyn Decision>, _request: Request| {
+ let started = started.clone();
+ async move {
+ if decision.selected_model() == "loser" {
+ started.notify_one();
+ match loser_delay {
+ Some(delay) => tokio::time::sleep(delay).await,
+ None => std::future::pending::<()>().await,
+ }
+ } else {
+ started.notified().await;
+ }
+ Ok(reply(decision.selected_model()))
+ }
};
- Arc::new(Hedge { winner, loser })
+ (algo, serve)
}
#[tokio::test]
async fn run_returns_the_winner_without_a_late_loser_overwriting_it() -> Result<()> {
// The loser responds 50ms after the winner has already won. `run` must return the
// winner, not the loser's `respond`-to-a-dropped-receiver error.
- let (_trace, response) = hedge(Some(std::time::Duration::from_millis(50)))
- .run(Context::default(), request())
- .await?;
+ let (algo, serve) = hedge(Some(std::time::Duration::from_millis(50)));
+ let (_trace, response) = drive(algo, Context::default(), request(), serve).await?;
assert_eq!(
response
.llm_response
@@ -1709,7 +1456,8 @@ mod tests {
async fn run_returns_the_winner_without_hanging_on_a_pending_loser() -> Result<()> {
// The loser never resolves. `run` must return the winner promptly, not hang
// waiting for the in-flight loser.
- let run = hedge(None).run(Context::default(), request());
+ let (algo, serve) = hedge(None);
+ let run = drive(algo, Context::default(), request(), serve);
let (_trace, response) = tokio::time::timeout(std::time::Duration::from_secs(1), run)
.await
.map_err(|error| LibsyError::external("waiting for pending loser", error))??;
@@ -1732,33 +1480,9 @@ mod tests {
// error must still reach the caller with all of these calls pending.
const N: usize = 10;
- // Enters each call; once all N are in flight, signals, then pends forever.
- struct EnterThenPend {
- started: Arc<AtomicUsize>,
- all_started: Arc<tokio::sync::Notify>,
- n: usize,
- }
-
- #[async_trait]
- impl RoutedLlmClient for EnterThenPend {
- async fn call(
- &self,
- _ctx: Context,
- _request: Request,
- _decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, LlmClientError> {
- if self.started.fetch_add(1, Ordering::SeqCst) + 1 == self.n {
- self.all_started.notify_one();
- }
- std::future::pending::<()>().await;
- unreachable!()
- }
- }
-
// Fans out N calls, then errors as soon as all N are in flight — exercising a
// terminal failure emitted while the offloaded calls are still pending.
struct FanOutThenError {
- target: LlmTarget,
all_started: Arc<tokio::sync::Notify>,
n: usize,
}
@@ -1779,7 +1503,11 @@ mod tests {
let decision: Arc<dyn Decision> = Arc::new(TestDecision {
model: format!("m{i}"),
});
- driver.call_llm_target(ctx.clone(), &self.target, request.clone(), decision)
+ driver.call_llm(RoutedRequest {
+ request: request.clone(),
+ decision,
+ ctx: ctx.clone(),
+ })
}));
tokio::select! {
_ = offloads => Err(test_error("offloads unexpectedly completed")),
@@ -1791,23 +1519,27 @@ mod tests {
}
let all_started = Arc::new(tokio::sync::Notify::new());
- let target = LlmTarget {
- semantic_name: "pending".to_string(),
- llm_client: Some(Arc::new(EnterThenPend {
- started: Arc::new(AtomicUsize::new(0)),
- all_started: all_started.clone(),
- n: N,
- })),
- };
let algo: Arc<dyn Algorithm> = Arc::new(FanOutThenError {
- target,
- all_started,
+ all_started: all_started.clone(),
n: N,
});
- // With the cap gone, `run` keeps polling the stream even with N calls in flight, so
- // the terminal error surfaces promptly instead of hanging.
- let run = algo.run(Context::default(), request());
+ // Serving enters each call; once all N are in flight it signals, then pends forever.
+ let started = Arc::new(AtomicUsize::new(0));
+ let serve = move |_decision: Arc<dyn Decision>, _request: Request| {
+ let started = started.clone();
+ let all_started = all_started.clone();
+ async move {
+ if started.fetch_add(1, Ordering::SeqCst) + 1 == N {
+ all_started.notify_one();
+ }
+ std::future::pending::<ServeResult>().await
+ }
+ };
+
+ // With the cap gone, the driver keeps polling the stream even with N calls in
+ // flight, so the terminal error surfaces promptly instead of hanging.
+ let run = drive(algo, Context::default(), request(), serve);
let result = tokio::time::timeout(std::time::Duration::from_millis(500), run)
.await
.map_err(|error| {
diff --git a/crates/libsy/src/core/testing.rs b/crates/libsy/src/core/testing.rs
new file mode 100644
index 000000000..9a0e4f7aa
--- /dev/null
+++ b/crates/libsy/src/core/testing.rs
@@ -0,0 +1,102 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Fake clients for libsy's own tests.
+//!
+//! libsy offloads every model call, so a test needs something to answer them. [`drive`] is
+//! [`crate::drive`] with the promise handling filled in: a test supplies a [`Serve`] closure
+//! standing in for the client a real host would use, and gets back the same
+//! `(trace, response)` pair a host does — the same path `switchyard-llm-client`'s `run`
+//! takes over HTTP.
+//!
+//! The closure is async so a fake can block on a barrier, wait on a notify, or never
+//! resolve, which is what the concurrency, hedging, and fan-out tests need.
+
+use std::future::Future;
+use std::sync::Arc;
+
+use futures::future::BoxFuture;
+use switchyard_protocol::{
+ Context, Decision, LlmClientError, LlmResponse, Request, Response, text_response,
+};
+
+use crate::core::algorithm::{Algorithm, CallLlmRequest, RunObserver};
+use crate::{LibsyError, Result};
+
+/// The result a fake client hands back for one offloaded call.
+pub(crate) type ServeResult = std::result::Result<Response, LlmClientError>;
+
+/// Answers offloaded model calls. Returning `Err` propagates a failed *model* call back into
+/// the algorithm, which may route around it.
+pub(crate) trait Serve: Send + Sync + 'static {
+ fn serve(
+ &self,
+ decision: Arc<dyn Decision>,
+ request: Request,
+ ) -> BoxFuture<'static, ServeResult>;
+}
+
+impl<F, Fut> Serve for F
+where
+ F: Fn(Arc<dyn Decision>, Request) -> Fut + Send + Sync + 'static,
+ Fut: Future<Output = ServeResult> + Send + 'static,
+{
+ fn serve(
+ &self,
+ decision: Arc<dyn Decision>,
+ request: Request,
+ ) -> BoxFuture<'static, ServeResult> {
+ Box::pin(self(decision, request))
+ }
+}
+
+/// Run `algorithm` to completion, serving each offloaded call with `serve`.
+pub(crate) async fn drive(
+ algorithm: Arc<dyn Algorithm>,
+ ctx: Context,
+ request: Request,
+ serve: impl Serve,
+) -> Result<(Vec<Arc<dyn Decision>>, Response)> {
+ drive_observed(algorithm, ctx, request, None, serve).await
+}
+
+/// [`drive`] plus a [`RunObserver`], for tests that assert on run observations.
+pub(crate) async fn drive_observed(
+ algorithm: Arc<dyn Algorithm>,
+ ctx: Context,
+ request: Request,
+ observer: Option<RunObserver>,
+ serve: impl Serve,
+) -> Result<(Vec<Arc<dyn Decision>>, Response)> {
+ let serve = Arc::new(serve);
+ crate::drive(algorithm, ctx, request, observer, move |call| {
+ fulfill(Arc::clone(&serve), call)
+ })
+ .await
+}
+
+/// Serve one call and fulfill its promise, mapping failures the way a host does so
+/// error-shape assertions match production.
+async fn fulfill(serve: Arc<impl Serve>, call: CallLlmRequest) -> Result<()> {
+ let routed = call.get_routed().clone();
+ let target = routed.decision.selected_model().to_string();
+ let result = serve
+ .serve(routed.decision, routed.request)
+ .await
+ .map_err(|source| LibsyError::client_call(target, source));
+ call.respond(result)
+}
+
+/// Answers with the selected model name as the completion — what most routing tests need,
+/// since they assert on *which* target was called.
+pub(crate) fn echo() -> impl Serve {
+ |decision: Arc<dyn Decision>, _request: Request| async move { Ok(reply(decision.selected_model())) }
+}
+
+/// A buffered response whose completion text is `completion`.
+pub(crate) fn reply(completion: impl Into<String>) -> Response {
+ Response {
+ llm_response: LlmResponse::Agg(text_response(None, completion.into())),
+ metadata: None,
+ }
+}
diff --git a/crates/libsy/src/error.rs b/crates/libsy/src/error.rs
index 834ceff33..648245d82 100644
--- a/crates/libsy/src/error.rs
+++ b/crates/libsy/src/error.rs
@@ -32,13 +32,6 @@ pub enum LibsyError {
message: String,
},
- /// A routed target had no default client for [`crate::Algorithm::run`].
- #[error("target {target:?} has no client to serve the call")]
- MissingClient {
- /// Target that could not be served.
- target: String,
- },
-
/// The type-erased offload driver could not complete an operation.
#[error(transparent)]
Driver(#[from] DriverError),
diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs
index 37f6cb22f..d6593d23f 100644
--- a/crates/libsy/src/lib.rs
+++ b/crates/libsy/src/lib.rs
@@ -7,7 +7,7 @@
mod core;
pub use core::algorithm::{
Algorithm, CallLlmRequest, Driver, LlmCallObservation, LlmTarget, LlmTargetSet, RoutedRequest,
- RunObservation, RunObserver, Step, StepStream,
+ RunObservation, RunObserver, Step, StepStream, drive,
};
pub use core::classifier::{Classification, Classifier, Score};
pub use core::processor::{Event, Processor};
@@ -41,6 +41,7 @@ pub use algorithms::util::stage::{
};
mod observability;
+pub use observability::algorithm_label;
/// Registers process-wide compatibility gauges with the global meter provider.
///
diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs
index bf3f1f1fe..1e02bcc88 100644
--- a/crates/libsy/src/observability.rs
+++ b/crates/libsy/src/observability.rs
@@ -4,9 +4,11 @@
//! OpenTelemetry metrics plus `tracing` spans and structured logs for the
//! algorithm layer.
//!
-//! The crate's provided run methods call these helpers around the [`Decision`]
-//! hook and the offload boundary, so every algorithm is instrumented from the
-//! outside and carries no telemetry code of its own. Metrics record through the
+//! [`Algorithm::run_stream`](crate::Algorithm::run_stream) and [`Driver`] call these
+//! helpers around the [`Decision`] hook and the offload boundary, so every algorithm is
+//! instrumented from the outside and carries no telemetry code of its own. The provider
+//! call on the other side of the offload belongs to the host, and is instrumented by
+//! whoever makes it. Metrics record through the
//! OpenTelemetry **global** meter provider under the `switchyard` scope — the host
//! installs an SDK provider and exporters; with none installed, recording is a
//! no-op. Spans and logs use the `tracing` facade (the async-native surface the
@@ -30,26 +32,17 @@
//! provider installed at any point in the process lifetime; the cost is
//! negligible next to a model call.
-use std::borrow::Cow;
use std::future::Future;
-use std::pin::Pin;
use std::sync::OnceLock;
use std::sync::atomic::{AtomicU64, Ordering};
-use std::task::{Context as TaskContext, Poll};
use std::time::{Duration, Instant};
-use futures::Stream;
use opentelemetry::metrics::{Meter, ObservableGauge};
-use opentelemetry::{Array as OtelArray, KeyValue, StringValue, Value as OtelValue, global};
-use switchyard_protocol::StopReason;
+use opentelemetry::{KeyValue, global};
use tracing::Span;
-use tracing_opentelemetry::OpenTelemetrySpanExt;
-use crate::{Driver, LibsyError, Result};
-use switchyard_protocol::{
- AggLlmResponse, Context, Decision, LlmClientError, LlmRequest, LlmResponse, LlmResponseChunk,
- LlmResponseStream, LlmResponseStreamEvent, Request, Response, Usage,
-};
+use crate::{Driver, Result};
+use switchyard_protocol::{Context, Decision, Request, Response};
const METRICS_SCOPE: &str = "switchyard";
const TRACING_TARGET: &str = "libsy";
@@ -62,8 +55,12 @@ static TOTAL_GAUGES: OnceLock<(ObservableGauge<u64>, ObservableGauge<u64>)> = On
/// telemetry label ([`Algorithm::name`](crate::Algorithm::name)).
pub(crate) const ALGORITHM_KEY: &str = "algorithm";
-/// The algorithm label carried by a request context; empty until stamped.
-pub(crate) fn algorithm_label(ctx: &Context) -> &str {
+/// The algorithm label [`Algorithm::run_stream`](crate::Algorithm::run_stream) stamps into
+/// a request context; empty until stamped.
+///
+/// A host instrumenting the calls it serves reads this to attribute its own spans to the
+/// algorithm that produced them.
+pub fn algorithm_label(ctx: &Context) -> &str {
ctx.values
.get(ALGORITHM_KEY)
.map(String::as_str)
@@ -171,295 +168,6 @@ pub(crate) async fn observe_run(
result
}
-/// Records request parameters represented directly by the neutral IR.
-pub(crate) fn record_gen_ai_request(span: &Span, request: &LlmRequest) {
- if request.stream {
- span.record("gen_ai.request.stream", true);
- }
- if let Some(value) = request.sampling.temperature {
- span.record("gen_ai.request.temperature", value);
- }
- if let Some(value) = request.sampling.top_p {
- span.record("gen_ai.request.top_p", value);
- }
- if let Some(value) = request.sampling.top_k {
- span.record("gen_ai.request.top_k", value);
- }
- if let Some(value) = request.output.max_output_tokens {
- span.record("gen_ai.request.max_tokens", otel_int(value));
- }
- if let Some(value) = request.reasoning.effort.as_deref() {
- span.record("gen_ai.request.reasoning.level", value);
- }
- if let Some(value) = request
- .output
- .response_format
- .as_ref()
- .and_then(gen_ai_output_type)
- {
- span.record("gen_ai.output.type", value);
- }
-}
-
-/// Adds terminal response and usage fields to the enclosing `libsy.client_call`
-/// span without consuming or buffering a streaming response.
-pub(crate) fn observe_client_call(result: Result<Response>) -> Result<Response> {
- let span = Span::current();
- match result {
- Ok(mut response) => {
- match response.llm_response {
- LlmResponse::Agg(agg) => {
- span.record("outcome", "ok");
- record_gen_ai_response(&span, &agg);
- response.llm_response = LlmResponse::Agg(agg);
- }
- LlmResponse::Stream(stream) => {
- response.llm_response =
- LlmResponse::Stream(observe_client_stream(stream, span));
- }
- }
- Ok(response)
- }
- Err(error) => {
- let error_type = client_call_error_type(&error);
- record_client_error(&span, &error_type, &error);
- Err(error)
- }
- }
-}
-
-fn observe_client_stream(stream: LlmResponseStream, span: Span) -> LlmResponseStream {
- Box::pin(ObservedClientStream {
- stream,
- observer: Some(ClientStreamObserver {
- span,
- terminal: false,
- }),
- })
-}
-
-fn record_gen_ai_response(span: &Span, response: &AggLlmResponse) {
- record_optional(span, "gen_ai.response.id", response.id.as_deref());
- record_optional(span, "gen_ai.response.model", response.model.as_deref());
- record_finish_reasons(
- span,
- response
- .outputs
- .iter()
- .filter_map(|output| output.stop_reason)
- .map(stop_reason_name),
- );
- record_gen_ai_usage(span, &response.usage);
-}
-
-fn record_gen_ai_usage(span: &Span, usage: &Usage) {
- let cache_read = usage.cached_input_tokens();
- let cache_creation = usage.cache_creation_input_tokens();
- if usage.input_tokens.is_some() || cache_read.is_some() || cache_creation.is_some() {
- let input_tokens = usage
- .input_tokens
- .unwrap_or_default()
- .saturating_add(cache_read.unwrap_or_default())
- .saturating_add(cache_creation.unwrap_or_default());
- span.record("gen_ai.usage.input_tokens", otel_int(input_tokens));
- }
- for (field, value) in [
- ("gen_ai.usage.output_tokens", usage.output_tokens),
- ("gen_ai.usage.cache_read.input_tokens", cache_read),
- ("gen_ai.usage.cache_creation.input_tokens", cache_creation),
- (
- "gen_ai.usage.reasoning.output_tokens",
- usage.reasoning_tokens,
- ),
- ] {
- if let Some(value) = value {
- span.record(field, otel_int(value));
- }
- }
-}
-
-// OpenTelemetry integer attributes are signed; token counts are unsigned in the IR.
-fn otel_int(value: u64) -> i64 {
- value.min(i64::MAX as u64) as i64
-}
-
-fn record_optional(span: &Span, field: &str, value: Option<&str>) {
- if let Some(value) = value {
- span.record(field, value);
- }
-}
-
-// `tracing` fields cannot preserve a typed string array, so write this attribute directly.
-fn record_finish_reasons(span: &Span, reasons: impl IntoIterator<Item = impl Into<String>>) {
- let reasons = reasons
- .into_iter()
- .map(|reason| StringValue::from(reason.into()))
- .collect::<Vec<_>>();
- if !reasons.is_empty() {
- span.set_attribute(
- "gen_ai.response.finish_reasons",
- OtelValue::Array(OtelArray::String(reasons)),
- );
- }
-}
-
-fn stop_reason_name(reason: StopReason) -> &'static str {
- match reason {
- StopReason::EndTurn => "end_turn",
- StopReason::MaxTokens => "max_tokens",
- StopReason::ToolUse => "tool_use",
- StopReason::ContentFilter => "content_filter",
- StopReason::Error => "error",
- StopReason::Unknown => "unknown",
- }
-}
-
-fn gen_ai_output_type(response_format: &serde_json::Value) -> Option<&'static str> {
- match response_format
- .get("type")
- .and_then(serde_json::Value::as_str)
- {
- Some("json" | "json_object" | "json_schema") => Some("json"),
- Some("text") => Some("text"),
- _ => None,
- }
-}
-
-fn client_call_error_type(error: &LibsyError) -> Cow<'static, str> {
- match error {
- LibsyError::ClientCall { source, .. } => llm_client_error_type(source),
- LibsyError::TargetNotFound { .. } => Cow::Borrowed("target_not_found"),
- LibsyError::NoTargets => Cow::Borrowed("no_targets"),
- LibsyError::AlgorithmError { .. } => Cow::Borrowed("algorithm_error"),
- LibsyError::MissingClient { .. } => Cow::Borrowed("client_configuration_error"),
- LibsyError::Driver(_) => Cow::Borrowed("driver_error"),
- LibsyError::AlgorithmTask { .. } => Cow::Borrowed("algorithm_task_error"),
- LibsyError::MissingFinalResponse => Cow::Borrowed("missing_final_response"),
- LibsyError::AllTargetsExcluded => Cow::Borrowed("context_window_exceeded"),
- LibsyError::External { .. } => Cow::Borrowed("_OTHER"),
- }
-}
-
-fn llm_client_error_type(error: &LlmClientError) -> Cow<'static, str> {
- match error {
- LlmClientError::InvalidRequest { .. } => Cow::Borrowed("invalid_request"),
- LlmClientError::RequestTranslation(_) => Cow::Borrowed("request_translation"),
- LlmClientError::RequestEncoding(_) => Cow::Borrowed("request_encoding"),
- LlmClientError::ResponseTranslation(_) => Cow::Borrowed("response_translation"),
- LlmClientError::Configuration { .. } => Cow::Borrowed("configuration"),
- LlmClientError::Transport { .. } => Cow::Borrowed("transport"),
- LlmClientError::Timeout { .. } => Cow::Borrowed("timeout"),
- LlmClientError::ContextWindowExceeded { .. } => Cow::Borrowed("context_window_exceeded"),
- LlmClientError::UpstreamHttp { status, .. } => Cow::Owned(status.to_string()),
- LlmClientError::InvalidResponse { .. } => Cow::Borrowed("invalid_response"),
- LlmClientError::Ffi { .. } => Cow::Borrowed("ffi"),
- _ => Cow::Borrowed("_OTHER"),
- }
-}
-
-// Keep the client span alive until the response is drained, errors, or is abandoned.
-struct ObservedClientStream {
- stream: LlmResponseStream,
- observer: Option<ClientStreamObserver>,
-}
-
-impl Stream for ObservedClientStream {
- type Item = std::result::Result<LlmResponseStreamEvent, LlmClientError>;
-
- fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Option<Self::Item>> {
- match self.stream.as_mut().poll_next(cx) {
- Poll::Ready(Some(item)) => {
- let failed = self
- .observer
- .as_mut()
- .is_some_and(|observer| observer.observe(&item));
- if failed {
- self.observer.take();
- }
- Poll::Ready(Some(item))
- }
- Poll::Ready(None) => {
- if let Some(mut observer) = self.observer.take() {
- observer.complete();
- }
- Poll::Ready(None)
- }
- Poll::Pending => Poll::Pending,
- }
- }
-}
-
-struct ClientStreamObserver {
- span: Span,
- terminal: bool,
-}
-
-impl ClientStreamObserver {
- fn observe(
- &mut self,
- item: &std::result::Result<LlmResponseStreamEvent, LlmClientError>,
- ) -> bool {
- match item {
- Ok(event) => {
- for chunk in event.normalized() {
- self.observe_chunk(chunk);
- if self.terminal {
- break;
- }
- }
- }
- Err(error) => {
- let error_type = llm_client_error_type(error);
- record_client_error(&self.span, &error_type, error);
- self.terminal = true;
- }
- }
- self.terminal
- }
-
- fn observe_chunk(&mut self, chunk: &LlmResponseChunk) {
- match chunk {
- LlmResponseChunk::MessageStart { id, model } => {
- record_optional(&self.span, "gen_ai.response.id", id.as_deref());
- record_optional(&self.span, "gen_ai.response.model", model.as_deref());
- }
- LlmResponseChunk::Usage(usage) => record_gen_ai_usage(&self.span, usage),
- LlmResponseChunk::MessageStop { reason } => {
- record_finish_reasons(&self.span, reason.iter().cloned());
- }
- LlmResponseChunk::DecodeError { message } => {
- record_client_error(&self.span, "response_translation", message);
- self.terminal = true;
- }
- LlmResponseChunk::StreamError { message } => {
- record_client_error(&self.span, "502", message);
- self.terminal = true;
- }
- _ => {}
- }
- }
-
- fn complete(&mut self) {
- self.span.record("outcome", "ok");
- self.terminal = true;
- }
-}
-
-impl Drop for ClientStreamObserver {
- fn drop(&mut self) {
- if !self.terminal {
- self.span.record("outcome", "cancelled");
- }
- }
-}
-
-fn record_client_error(span: &Span, error_type: &str, error: &dyn std::fmt::Display) {
- span.record("outcome", "error");
- span.record("otel.status_code", "ERROR");
- span.record("error.type", error_type);
- span.record("error", tracing::field::display(error));
-}
-
/// Records the end of one algorithm run: the run counter and duration
/// histogram, the `outcome`/`error` fields on `span`, and a warn log when the
/// run failed.
diff --git a/crates/switchyard-py/Cargo.toml b/crates/switchyard-py/Cargo.toml
index 378e1517a..ceaa7cf7f 100644
--- a/crates/switchyard-py/Cargo.toml
+++ b/crates/switchyard-py/Cargo.toml
@@ -21,6 +21,7 @@ futures-util.workspace = true
http.workspace = true
parking_lot.workspace = true
switchyard-libsy.workspace = true
+switchyard-llm-client.workspace = true
pyo3 = { version = "0.28.3", features = ["abi3-py312", "extension-module"] }
pyo3-async-runtimes = { version = "0.28", features = ["tokio-runtime"] }
pythonize = "0.28.0"
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index 7f13746ad..d33934706 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -14,6 +14,7 @@ use switchyard_libsy::{
LlmClassifierConfig, LlmFallback, LlmTarget, LlmTargetSet, LlmTaskClassifier, Noop, PickerMode,
Random, StageRouter, StageRouterConfig, TaskClassifierConfig,
};
+use switchyard_llm_client::ClientRouter;
use switchyard_protocol::{
AggLlmResponse, Context, Decision, LlmClientError, LlmResponse, Metadata, Request, Response,
RoutedLlmClient,
@@ -62,14 +63,23 @@ struct PyLlmTarget {
}
impl PyLlmTarget {
- fn clone_core(&self, py: Python<'_>) -> LlmTarget {
+ fn clone_core(&self, _py: Python<'_>) -> LlmTarget {
LlmTarget {
semantic_name: self.name.clone(),
- llm_client: Some(Arc::new(PythonLlmClient {
- inner: self.client.clone_ref(py),
- })),
}
}
+
+ /// The `selected_model -> client` entry this target contributes to the algorithm's
+ /// [`ClientRouter`]. libsy no longer carries the client, so the bindings keep the
+ /// mapping and serve the calls themselves.
+ fn client_entry(&self, py: Python<'_>) -> ClientEntry {
+ (
+ self.name.clone(),
+ Arc::new(PythonLlmClient {
+ inner: self.client.clone_ref(py),
+ }),
+ )
+ }
}
#[pymethods]
@@ -170,6 +180,11 @@ struct PyLlmFallback {
}
impl PyLlmFallback {
+ /// The judge's client entry, so the caller can register it with the algorithm's router.
+ fn judge_client_entry(&self, py: Python<'_>) -> PyResult<ClientEntry> {
+ Ok(self.judge_target.bind(py).try_borrow()?.client_entry(py))
+ }
+
fn clone_core(&self, py: Python<'_>) -> PyResult<LlmFallback> {
Ok(LlmFallback {
judge_target: self.judge_target.bind(py).try_borrow()?.clone_core(py),
@@ -194,11 +209,19 @@ impl PyLlmFallback {
#[pyclass(name = "Algorithm", module = "switchyard.libsy", frozen)]
struct PyAlgorithm {
inner: Arc<dyn Algorithm>,
+ /// Resolves the calls `inner` offloads to each target's Python client.
+ client_router: ClientRouter,
}
+/// One target's `selected_model -> client` mapping for an algorithm's router.
+type ClientEntry = (String, Arc<dyn RoutedLlmClient>);
+
impl PyAlgorithm {
- fn new(inner: Arc<dyn Algorithm>) -> Self {
- Self { inner }
+ fn new(inner: Arc<dyn Algorithm>, clients: impl IntoIterator<Item = ClientEntry>) -> Self {
+ Self {
+ inner,
+ client_router: clients.into_iter().collect(),
+ }
}
}
@@ -218,6 +241,7 @@ impl PyAlgorithm {
headers: Option<std::collections::HashMap<String, String>>,
) -> PyResult<Bound<'py, PyAny>> {
let algorithm = Arc::clone(&self.inner);
+ let client_router = self.client_router.clone();
let headers = headers.as_ref().map(header_map_from_python).transpose()?;
let request = Request {
@@ -226,10 +250,15 @@ impl PyAlgorithm {
metadata: headers.map(|headers| Metadata::from_headers(&headers)),
};
pyo3_async_runtimes::tokio::future_into_py(py, async move {
- let (decisions, response) = algorithm
- .run(Context::default(), request)
- .await
- .map_err(py_libsy_error)?;
+ let (decisions, response) = switchyard_llm_client::run(
+ algorithm,
+ client_router,
+ Context::default(),
+ request,
+ None,
+ )
+ .await
+ .map_err(py_libsy_error)?;
let response = response
.llm_response
.into_agg()
@@ -256,7 +285,8 @@ impl PyAlgorithm {
/// Construct the no-op reference algorithm.
#[pyfunction(name = "noop")]
fn noop_algorithm() -> PyAlgorithm {
- PyAlgorithm::new(Arc::new(Noop {}))
+ // `Noop` synthesizes its own response and never offloads a call, so it needs no clients.
+ PyAlgorithm::new(Arc::new(Noop {}), [])
}
/// Construct random routing over targets with optional relative weights and seed.
@@ -268,18 +298,31 @@ fn random_algorithm(
weights: Option<Vec<f64>>,
seed: Option<u64>,
) -> PyResult<PyAlgorithm> {
- let targets = targets
- .iter()
- .map(|target| Ok(target.bind(py).try_borrow()?.clone_core(py)))
- .collect::<PyResult<Vec<_>>>()?;
+ let (cores, clients) = target_cores(py, &targets)?;
let algorithm =
- Random::new(LlmTargetSet::new(targets), weights, seed).map_err(|error| match error {
+ Random::new(LlmTargetSet::new(cores), weights, seed).map_err(|error| match error {
RustLibsyError::NoTargets => {
PyValueError::new_err("random requires at least one target")
}
other => PyValueError::new_err(other.to_string()),
})?;
- Ok(PyAlgorithm::new(Arc::new(algorithm)))
+ Ok(PyAlgorithm::new(Arc::new(algorithm), clients))
+}
+
+/// Splits a list of Python targets into libsy's client-free targets and the client entries
+/// the bindings keep for the algorithm's router.
+fn target_cores(
+ py: Python<'_>,
+ targets: &[Py<PyLlmTarget>],
+) -> PyResult<(Vec<LlmTarget>, Vec<ClientEntry>)> {
+ let mut cores = Vec::with_capacity(targets.len());
+ let mut clients = Vec::with_capacity(targets.len());
+ for target in targets {
+ let target = target.bind(py).try_borrow()?;
+ cores.push(target.clone_core(py));
+ clients.push(target.client_entry(py));
+ }
+ Ok((cores, clients))
}
/// Construct task-level LLM classifier routing.
@@ -298,14 +341,21 @@ fn llm_task_classifier_algorithm(
capable_target: Py<PyLlmTarget>,
config: Py<PyTaskClassifierConfig>,
) -> PyResult<PyAlgorithm> {
+ let (cores, clients) = target_cores(
+ py,
+ &[judge_target.clone_ref(py), efficient_target, capable_target],
+ )?;
+ let [judge, efficient, capable] = cores
+ .try_into()
+ .map_err(|_| PyValueError::new_err("expected three targets"))?;
let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Capability {
- judge_target: judge_target.bind(py).try_borrow()?.clone_core(py),
- efficient_target: efficient_target.bind(py).try_borrow()?.clone_core(py),
- capable_target: capable_target.bind(py).try_borrow()?.clone_core(py),
+ judge_target: judge,
+ efficient_target: efficient,
+ capable_target: capable,
config: config.bind(py).try_borrow()?.clone_core(),
})
.map_err(|error| PyValueError::new_err(error.to_string()))?;
- Ok(PyAlgorithm::new(Arc::new(algorithm)))
+ Ok(PyAlgorithm::new(Arc::new(algorithm), clients))
}
/// Construct signal-driven stage routing with an optional LLM classifier fallback.
@@ -348,8 +398,10 @@ fn stage_router_algorithm(
)));
}
};
- let capable = capable_target.bind(py).try_borrow()?.clone_core(py);
- let efficient = efficient_target.bind(py).try_borrow()?.clone_core(py);
+ let (cores, mut clients) = target_cores(py, &[capable_target, efficient_target])?;
+ let [capable, efficient] = cores
+ .try_into()
+ .map_err(|_| PyValueError::new_err("expected two targets"))?;
let mut config = StageRouterConfig::new(mode, confidence_threshold);
config.recent_window = recent_window;
config.handoff_notes = match (escalation_note, deescalation_note) {
@@ -375,13 +427,18 @@ fn stage_router_algorithm(
.tier_prompts
.with(efficient.semantic_name.clone(), prompt);
}
+ // The judge is only reachable through the optional classifier fallback, so its client
+ // joins the router only when a fallback is configured.
+ if let Some(classifier) = &classifier {
+ clients.push(classifier.bind(py).try_borrow()?.judge_client_entry(py)?);
+ }
config.llm_fallback = classifier
.map(|classifier| classifier.bind(py).try_borrow()?.clone_core(py))
.transpose()?;
let algorithm = StageRouter::new(capable, efficient, config)
.map_err(|error| PyValueError::new_err(error.to_string()))?;
- Ok(PyAlgorithm::new(Arc::new(algorithm)))
+ Ok(PyAlgorithm::new(Arc::new(algorithm), clients))
}
fn other_python_error(error: PyErr) -> LlmClientError {
diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.toml
index 15391a98b..5d8a44939 100644
--- a/crates/switchyard-server/Cargo.toml
+++ b/crates/switchyard-server/Cargo.toml
@@ -36,8 +36,8 @@ switchyard-translation.workspace = true
humantime = "2.4"
tokio.workspace = true
tracing.workspace = true
-tracing-opentelemetry = "0.33"
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+tracing-opentelemetry.workspace = true
+tracing-subscriber.workspace = true
[dev-dependencies]
async-trait.workspace = true
diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs
index 676efdf25..b644d4a04 100644
--- a/crates/switchyard-server/src/config.rs
+++ b/crates/switchyard-server/src/config.rs
@@ -17,7 +17,8 @@ use libsy::{
use serde::Deserialize;
use serde_json::Value;
use switchyard_llm_client::{
- Backend, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig, TranslatingLlmClient,
+ Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig,
+ TranslatingLlmClient,
};
use switchyard_protocol::RoutedLlmClient;
@@ -86,7 +87,7 @@ impl ServerConfig {
}
let clients = self.build_clients()?;
- let targets = self.build_targets(&clients)?;
+ let targets = self.build_targets()?;
let mut routes = Vec::with_capacity(self.routes.len());
for (route_name, config) in &self.routes {
validate_value("route name", route_name)?;
@@ -98,10 +99,12 @@ impl ServerConfig {
)));
}
let algorithm = build_algorithm(route_name, config, &targets)?;
+ let client = self.build_client_router(config, &clients)?;
let count_tokens_target = self.build_count_tokens_target(config, &clients);
routes.push((
config.id().to_string(),
algorithm,
+ client,
capabilities,
count_tokens_target,
));
@@ -147,26 +150,51 @@ impl ServerConfig {
Ok(clients)
}
- fn build_targets(
- &self,
- clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,
- ) -> ServerResult<BTreeMap<String, LlmTarget>> {
- self.targets
+ fn build_targets(&self) -> ServerResult<BTreeMap<String, LlmTarget>> {
+ Ok(self
+ .targets
.iter()
.map(|(name, config)| {
- let client = clients.get(&config.llm_client).ok_or_else(|| {
- ServerError::new(format!("target {name} has no constructed llm client"))
- })?;
- let client: Arc<dyn RoutedLlmClient> = client.clone();
- Ok((
+ (
name.clone(),
LlmTarget {
semantic_name: config.id.clone(),
- llm_client: Some(client),
},
- ))
+ )
})
- .collect()
+ .collect())
+ }
+
+ /// Maps every model this route can select to the client configured for that target.
+ ///
+ /// A route is a synthetic model (`switchyard/random`) with no upstream of its own — the
+ /// algorithm picks a real target, and the *target* names the `[llm_clients.*]` section
+ /// that serves it. Two targets in one route may therefore sit on different clients, and
+ /// one request can emit calls for both.
+ ///
+ /// The map is built per route because targets are only reachable through the routes that
+ /// name them, so the same model id may resolve to different clients in two routes without
+ /// colliding.
+ fn build_client_router(
+ &self,
+ route: &RouteConfig,
+ clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,
+ ) -> ServerResult<ClientRouter> {
+ let by_model = route
+ .callable_target_names()
+ .into_iter()
+ .map(|name| {
+ let target = self.targets.get(name).ok_or_else(|| {
+ ServerError::new(format!("route references unknown target {name}"))
+ })?;
+ let client = clients.get(&target.llm_client).ok_or_else(|| {
+ ServerError::new(format!("target {name} has no constructed llm client"))
+ })?;
+ let client: Arc<dyn RoutedLlmClient> = client.clone();
+ Ok((target.id.clone(), client))
+ })
+ .collect::<ServerResult<_>>()?;
+ Ok(ClientRouter::new(by_model))
}
fn build_count_tokens_target(
@@ -490,6 +518,25 @@ impl RouteConfig {
}
}
+ /// Every target the algorithm may call, including judge-only targets.
+ ///
+ /// [`routing_target_names`](Self::routing_target_names) covers completion destinations;
+ /// a classifier also calls its judge, and that call needs a client too.
+ fn callable_target_names(&self) -> Vec<&str> {
+ let mut names = self.routing_target_names();
+ match self {
+ Self::LlmClassifier {
+ classifier_target, ..
+ } => names.push(classifier_target),
+ Self::StageRouter {
+ classifier: Some(classifier),
+ ..
+ } => names.push(&classifier.target),
+ _ => {}
+ }
+ names
+ }
+
fn capabilities(&self) -> ModelCapabilities {
use RouteConfig::*;
match self {
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index 8f5c03461..4e4b28993 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -36,7 +36,7 @@ use libsy::{Algorithm, LibsyError, RunObservation, RunObserver};
use parking_lot::Mutex;
use serde::Deserialize;
use serde_json::{Value, json};
-use switchyard_llm_client::TranslatingLlmClient;
+use switchyard_llm_client::{ClientRouter, TranslatingLlmClient};
use switchyard_protocol::{Context, Decision, LlmClientError, Metadata, Request, Usage};
use tokio::net::{TcpListener, TcpSocket};
use tokio::task;
@@ -106,6 +106,10 @@ struct ModelCapabilities {
/// A registered algorithm route and its server-owned endpoint metadata.
struct RouteEntry {
algorithm: Arc<dyn Algorithm>,
+ /// Resolves each offloaded call to the client configured for the target the algorithm
+ /// selected. A route is a synthetic model with no upstream of its own, so this is a
+ /// per-target lookup, never one client serving the whole route.
+ target_clients: ClientRouter,
capabilities: ModelCapabilities,
count_tokens_target: Option<CountTokensTarget>,
}
@@ -168,15 +172,20 @@ impl SharedRoutingLog {
}
impl ServerState {
- /// Creates server state from route model IDs and their libsy algorithms.
+ /// Creates server state from route model IDs, their libsy algorithms, and the
+ /// per-target client routing each route's calls are resolved through.
pub fn new(
- routes: impl IntoIterator<Item = (String, Arc<dyn Algorithm>)>,
+ routes: impl IntoIterator<Item = (String, Arc<dyn Algorithm>, ClientRouter)>,
) -> ServerResult<Self> {
- Self::new_with_capabilities(
- routes
- .into_iter()
- .map(|(model, algorithm)| (model, algorithm, ModelCapabilities::default(), None)),
- )
+ Self::new_with_capabilities(routes.into_iter().map(|(model, algorithm, clients)| {
+ (
+ model,
+ algorithm,
+ clients,
+ ModelCapabilities::default(),
+ None,
+ )
+ }))
}
fn new_with_capabilities(
@@ -184,19 +193,21 @@ impl ServerState {
Item = (
String,
Arc<dyn Algorithm>,
+ ClientRouter,
ModelCapabilities,
Option<CountTokensTarget>,
),
>,
) -> ServerResult<Self> {
let mut entries = BTreeMap::new();
- for (model, algorithm, capabilities, count_tokens_target) in routes {
+ for (model, algorithm, target_clients, capabilities, count_tokens_target) in routes {
let model = model.trim();
if model.is_empty() {
return Err(ServerError::new("route model must not be empty"));
}
let entry = RouteEntry {
algorithm,
+ target_clients,
capabilities,
count_tokens_target,
};
@@ -678,13 +689,21 @@ async fn handle_llm_request(
Err(response) => return response,
};
let algorithm = Arc::clone(&route.algorithm);
+ // Not one client for the route: a router that resolves each offloaded call to the
+ // client configured for the target the algorithm selected.
+ let target_clients = route.target_clients.clone();
let observer = stats_observer(
state.stats.clone(),
state.routing_log.clone().zip(routing_log_context.clone()),
);
- let (trace, response) = match algorithm
- .run_observed(Context::default(), request, Some(observer))
- .await
+ let (trace, response) = match switchyard_llm_client::run(
+ algorithm,
+ target_clients,
+ Context::default(),
+ request,
+ Some(observer),
+ )
+ .await
{
Ok(result) => result,
Err(error) => return algorithm_error(error),
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 3f3907558..93f11f986 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -19,7 +19,9 @@ use axum::{Json, Router};
use http_body_util::BodyExt;
use libsy::{Algorithm, LlmTarget, LlmTargetSet, Random};
use serde_json::{Value, json};
-use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
+use switchyard_llm_client::{
+ Backend, ClientRouter, HttpBackendConfig, ModelConfig, TranslatingLlmClient,
+};
use switchyard_protocol::RoutedLlmClient;
use switchyard_server::config::load_server_state;
use switchyard_server::{ServerState, build_switchyard_router};
@@ -61,6 +63,16 @@ impl MockUpstream {
task,
})
}
+
+ /// The upstream model id of every request this upstream received, in order.
+ async fn models(&self) -> Vec<String> {
+ self.calls
+ .lock()
+ .await
+ .iter()
+ .filter_map(|call| call.get("model")?.as_str().map(str::to_string))
+ .collect()
+ }
}
impl Drop for MockUpstream {
@@ -220,12 +232,15 @@ fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<Server
.iter()
.map(|model| LlmTarget {
semantic_name: (*model).to_string(),
- llm_client: Some(Arc::clone(&client)),
})
.collect(),
);
let algorithm: Arc<dyn Algorithm> = Arc::new(Random::new(target_set, None, None)?);
- Ok(((*route_model).to_string(), algorithm))
+ Ok((
+ (*route_model).to_string(),
+ algorithm,
+ ClientRouter::single(Arc::clone(&client)),
+ ))
})
.collect::<TestResult<Vec<_>>>()?;
Ok(ServerState::new(entries)?)
@@ -647,6 +662,84 @@ fn assert_in_order(haystack: &str, needles: &[&str]) {
}
}
+/// A route is a synthetic model (`switchyard/classify`) with no upstream of its own; its
+/// algorithm picks real targets, and *those* name the client. One request can emit several
+/// `Step::CallLlm` for different targets, and two targets may sit on different
+/// `[llm_clients.*]` sections — here the judge is on one provider and the serving models on
+/// another. Pin that each call reaches its own target's upstream, rather than one client
+/// chosen per route serving all of them.
+#[tokio::test]
+async fn each_target_in_one_request_is_served_by_its_own_client() -> TestResult {
+ let judge_upstream = MockUpstream::start().await?;
+ let model_upstream = MockUpstream::start().await?;
+ let state = load_test_config(&format!(
+ r#"
+schema_version = 1
+
+[llm_clients.judge_provider]
+format = "openai_chat"
+base_url = "{judge_url}"
+
+[llm_clients.model_provider]
+format = "openai_chat"
+base_url = "{model_url}"
+
+[targets.judge]
+id = "model/judge"
+llm_client = "judge_provider"
+
+[targets.strong]
+id = "model/strong"
+llm_client = "model_provider"
+
+[targets.weak]
+id = "model/weak"
+llm_client = "model_provider"
+
+[routes.classify]
+id = "switchyard/classify"
+type = "llm_classifier"
+classifier_target = "judge"
+strong_target = "strong"
+weak_target = "weak"
+base_threshold = 0.5
+"#,
+ judge_url = judge_upstream.base_url,
+ model_url = model_upstream.base_url,
+ ))?;
+ let app = build_switchyard_router(state);
+
+ let response = send(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({
+ "model": "switchyard/classify",
+ "messages": [{"role": "user", "content": "hi"}]
+ })),
+ )
+ .await?;
+ assert_eq!(response.status, StatusCode::OK);
+
+ // The judge call went to the judge's provider and nowhere else; the serving call went to
+ // the models' provider. A single per-route client would have sent both to one upstream.
+ assert_eq!(
+ judge_upstream.models().await,
+ vec!["model/judge".to_string()]
+ );
+ let served = model_upstream.models().await;
+ assert_eq!(
+ served.len(),
+ 1,
+ "expected exactly one routed call: {served:?}"
+ );
+ assert!(
+ served[0] == "model/weak" || served[0] == "model/strong",
+ "routed call went to {served:?}"
+ );
+ Ok(())
+}
+
/// A critical tool error must reach the stage router's signal scorer, which reads
/// the decoded conversation. The endpoint records no inbound wire format, so a
/// scorer that parsed the raw body instead would find nothing and route every turn