Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat: Move libsy::algorithm::run to libsy-llm-client (#324)
Graham King74b899054393
diff --git a/Cargo.lock b/Cargo.lockindex 09354946e..e5e151e0a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2298,7 +2298,6 @@ dependencies = ["rand 0.10.2","serde","serde_json",- "switchyard-llm-client","switchyard-protocol","thiserror 2.0.18","tokio",@@ -2318,12 +2317,18 @@ dependencies = ["http","httpdate","opentelemetry",+ "opentelemetry_sdk",+ "parking_lot","reqwest","serde_json",+ "switchyard-libsy","switchyard-protocol","switchyard-translation",+ "thiserror 2.0.18","tokio","tracing",+ "tracing-opentelemetry",+ "tracing-subscriber","wiremock",]@@ -2354,6 +2359,7 @@ dependencies = ["serde_json","switchyard-components","switchyard-libsy",+ "switchyard-llm-client","switchyard-protocol","switchyard-server","switchyard-translation",diff --git a/Cargo.toml b/Cargo.tomlindex 90c27d543..0c05c4767 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -45,3 +45,5 @@ switchyard-translation = { path = "crates/switchyard-translation", version = "0.thiserror = "2"tokio = { version = "1", features = ["full"] }tracing = { version = "0.1", default-features = false, features = ["attributes", "std"] }+tracing-opentelemetry = "0.33"+tracing-subscriber = { version = "0.3", features = ["env-filter"] }diff --git a/crates/libsy-llm-client/Cargo.toml b/crates/libsy-llm-client/Cargo.tomlindex 1f5ecc873..359019aef 100644--- a/crates/libsy-llm-client/Cargo.toml+++ b/crates/libsy-llm-client/Cargo.toml@@ -12,10 +12,12 @@ repository.workspace = truerust-version.workspace = true[dependencies]+switchyard-libsy.workspace = trueswitchyard-protocol.workspace = trueswitchyard-translation.workspace = truereqwest.workspace = trueasync-trait.workspace = true+futures.workspace = truefutures-util.workspace = trueopentelemetry = { version = "0.32", default-features = false, features = ["metrics"] }http.workspace = true@@ -23,7 +25,12 @@ httpdate.workspace = trueserde_json.workspace = truetokio.workspace = truetracing.workspace = true+tracing-opentelemetry.workspace = true[dev-dependencies]-futures.workspace = true+# SDK + in-memory exporter to assert what the observability layer records.+opentelemetry_sdk = { version = "0.32", features = ["metrics", "testing", "trace"] }+parking_lot.workspace = true+thiserror.workspace = true+tracing-subscriber.workspace = truewiremock = "0.6"diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.mdindex a1d5bbbc4..1ce9fbf71 100644--- a/crates/libsy-llm-client/README.md+++ b/crates/libsy-llm-client/README.md@@ -11,8 +11,12 @@ encodes the request to that backend's wire format, adds auth and forwards yourheaders, makes the call with a shared `reqwest::Client`, and decodes the replyback into a [`switchyard_protocol::Response`] — buffered or streamed.-It depends only on `switchyard-protocol` and `switchyard-translation`; no server, no-provider SDK.+It also pairs the client with a libsy algorithm: [`run`] drives+[`Algorithm::run_stream`] and serves every model call the algorithm offloads, so a+host that just wants the answer never has to drive the step stream itself.++It depends on `switchyard-libsy`, `switchyard-protocol`, and+`switchyard-translation`; no server, no provider SDK.## Concepts@@ -132,6 +136,42 @@ async fn stream(}```+### Routing an algorithm++[`run`] takes a libsy algorithm and a [`ClientRouter`], and returns the final response plus+the trace of decisions the algorithm published. The router resolves each offloaded call to+the client for the target the algorithm selected; `ClientRouter::single` is the+single-provider case:++```rust+use std::sync::Arc;+use switchyard_libsy::Algorithm;+use switchyard_llm_client::{ClientRouter, TranslatingLlmClient};+use switchyard_protocol::{Context, Request};++async fn route(+ algorithm: Arc<dyn Algorithm>,+ client: Arc<TranslatingLlmClient>,+ request: Request,+) -> switchyard_libsy::Result<String> {+ let clients = ClientRouter::single(client);+ let (trace, _response) =+ switchyard_llm_client::run(algorithm, clients, Context::default(), request, None).await?;+ Ok(trace+ .last()+ .map(|decision| decision.selected_model().to_string())+ .unwrap_or_default())+}+```++When targets are served by different clients — a judge on one provider and the serving+models on another, say — build the router from `model name -> client` with+[`ClientRouter::new`] instead. A model missing from that map fails with+`LlmClientError::Configuration` rather than silently going to another provider.++A router is not itself a client: [`ClientRouter::route`] hands back a+[`switchyard_protocol::RoutedLlmClient`] and the caller makes the call.+## Cross-format translationThe request/response are translated through the neutral IR, so the inbound shapediff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rsindex 920f2f3d6..f1c8a462d 100644--- a/crates/libsy-llm-client/src/lib.rs+++ b/crates/libsy-llm-client/src/lib.rs@@ -10,15 +10,23 @@//! HTTP call with a shared [`reqwest::Client`], and decodes the wire response//! back to a [`switchyard_protocol::Response`] — supporting both buffered and//! streamed responses.+//!+//! [`run()`] pairs the client with a libsy algorithm: it drives+//! [`switchyard_libsy::Algorithm::run_stream`] and serves every model call the algorithm+//! offloads, so a host that just wants the answer does not have to drive the step stream+//! itself.pub mod backend;pub mod client;pub mod error;pub mod metrics;+mod observability;pub mod raw;+pub mod run;pub use backend::{Backend, DEFAULT_MAX_RETRIES, HttpBackendConfig};pub use client::{ModelConfig, TranslatingLlmClient};pub use error::{LlmClientError, Result};pub use raw::RawResponse;+pub use run::{ClientRouter, run};pub use switchyard_translation::RawEventStream;diff --git a/crates/libsy-llm-client/src/observability.rs b/crates/libsy-llm-client/src/observability.rsnew file mode 100644index 000000000..1767ffad8--- /dev/null+++ b/crates/libsy-llm-client/src/observability.rs@@ -0,0 +1,335 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! `tracing` span fields for the provider call [`run`](crate::run) makes.+//!+//! libsy instruments the orchestration layer (`libsy.run`, `libsy.llm_call`) and owns the+//! `switchyard.*` metrics. This module owns the other half: the `libsy.client_call` span that+//! covers the one HTTP call a routed request performs, recorded as OpenTelemetry gen_ai+//! semantic-convention attributes.+//!+//! Nothing here records a metric — these are span fields only, written through the `tracing`+//! facade so the host's subscriber decides where they go. A streamed response resolves before+//! its usage is known, so [`observe_client_call`] wraps the stream and keeps the span alive+//! until it drains, errors, or is abandoned.++use std::borrow::Cow;+use std::pin::Pin;+use std::task::{Context as TaskContext, Poll};++use futures::Stream;+use opentelemetry::{Array as OtelArray, StringValue, Value as OtelValue};+use switchyard_libsy::{LibsyError, Result};+use switchyard_protocol::{+ AggLlmResponse, LlmClientError, LlmRequest, LlmResponse, LlmResponseChunk, LlmResponseStream,+ LlmResponseStreamEvent, Response, StopReason, Usage,+};+use tracing::Span;+use tracing_opentelemetry::OpenTelemetrySpanExt;++/// Records request parameters represented directly by the neutral IR.+pub(crate) fn record_gen_ai_request(span: &Span, request: &LlmRequest) {+ if request.stream {+ span.record("gen_ai.request.stream", true);+ }+ if let Some(value) = request.sampling.temperature {+ span.record("gen_ai.request.temperature", value);+ }+ if let Some(value) = request.sampling.top_p {+ span.record("gen_ai.request.top_p", value);+ }+ if let Some(value) = request.sampling.top_k {+ span.record("gen_ai.request.top_k", value);+ }+ if let Some(value) = request.output.max_output_tokens {+ span.record("gen_ai.request.max_tokens", otel_int(value));+ }+ if let Some(value) = request.reasoning.effort.as_deref() {+ span.record("gen_ai.request.reasoning.level", value);+ }+ if let Some(value) = request+ .output+ .response_format+ .as_ref()+ .and_then(gen_ai_output_type)+ {+ span.record("gen_ai.output.type", value);+ }+}++/// Adds terminal response and usage fields to the enclosing `libsy.client_call`+/// span without consuming or buffering a streaming response.+pub(crate) fn observe_client_call(result: Result<Response>) -> Result<Response> {+ let span = Span::current();+ match result {+ Ok(mut response) => {+ match response.llm_response {+ LlmResponse::Agg(agg) => {+ span.record("outcome", "ok");+ record_gen_ai_response(&span, &agg);+ response.llm_response = LlmResponse::Agg(agg);+ }+ LlmResponse::Stream(stream) => {+ response.llm_response =+ LlmResponse::Stream(observe_client_stream(stream, span));+ }+ }+ Ok(response)+ }+ Err(error) => {+ let error_type = client_call_error_type(&error);+ record_client_error(&span, &error_type, &error);+ Err(error)+ }+ }+}++fn observe_client_stream(stream: LlmResponseStream, span: Span) -> LlmResponseStream {+ Box::pin(ObservedClientStream {+ stream,+ observer: Some(ClientStreamObserver {+ span,+ outcome: Outcome::Open,+ }),+ })+}++fn record_gen_ai_response(span: &Span, response: &AggLlmResponse) {+ record_optional(span, "gen_ai.response.id", response.id.as_deref());+ record_optional(span, "gen_ai.response.model", response.model.as_deref());+ record_finish_reasons(+ span,+ response+ .outputs+ .iter()+ .filter_map(|output| output.stop_reason)+ .map(stop_reason_name),+ );+ record_gen_ai_usage(span, &response.usage);+}++fn record_gen_ai_usage(span: &Span, usage: &Usage) {+ let cache_read = usage.cached_input_tokens();+ let cache_creation = usage.cache_creation_input_tokens();+ if usage.input_tokens.is_some() || cache_read.is_some() || cache_creation.is_some() {+ let input_tokens = usage+ .input_tokens+ .unwrap_or_default()+ .saturating_add(cache_read.unwrap_or_default())+ .saturating_add(cache_creation.unwrap_or_default());+ span.record("gen_ai.usage.input_tokens", otel_int(input_tokens));+ }+ for (field, value) in [+ ("gen_ai.usage.output_tokens", usage.output_tokens),+ ("gen_ai.usage.cache_read.input_tokens", cache_read),+ ("gen_ai.usage.cache_creation.input_tokens", cache_creation),+ (+ "gen_ai.usage.reasoning.output_tokens",+ usage.reasoning_tokens,+ ),+ ] {+ if let Some(value) = value {+ span.record(field, otel_int(value));+ }+ }+}++// OpenTelemetry integer attributes are signed; token counts are unsigned in the IR.+fn otel_int(value: u64) -> i64 {+ value.min(i64::MAX as u64) as i64+}++fn record_optional(span: &Span, field: &str, value: Option<&str>) {+ if let Some(value) = value {+ span.record(field, value);+ }+}++// `tracing` fields cannot preserve a typed string array, so write this attribute directly.+fn record_finish_reasons(span: &Span, reasons: impl IntoIterator<Item = impl Into<String>>) {+ let reasons = reasons+ .into_iter()+ .map(|reason| StringValue::from(reason.into()))+ .collect::<Vec<_>>();+ if !reasons.is_empty() {+ span.set_attribute(+ "gen_ai.response.finish_reasons",+ OtelValue::Array(OtelArray::String(reasons)),+ );+ }+}++fn stop_reason_name(reason: StopReason) -> &'static str {+ match reason {+ StopReason::EndTurn => "end_turn",+ StopReason::MaxTokens => "max_tokens",+ StopReason::ToolUse => "tool_use",+ StopReason::ContentFilter => "content_filter",+ StopReason::Error => "error",+ StopReason::Unknown => "unknown",+ }+}++fn gen_ai_output_type(response_format: &serde_json::Value) -> Option<&'static str> {+ match response_format+ .get("type")+ .and_then(serde_json::Value::as_str)+ {+ Some("json" | "json_object" | "json_schema") => Some("json"),+ Some("text") => Some("text"),+ _ => None,+ }+}++fn client_call_error_type(error: &LibsyError) -> Cow<'static, str> {+ match error {+ LibsyError::ClientCall { source, .. } => llm_client_error_type(source),+ LibsyError::TargetNotFound { .. } => Cow::Borrowed("target_not_found"),+ LibsyError::NoTargets => Cow::Borrowed("no_targets"),+ LibsyError::AlgorithmError { .. } => Cow::Borrowed("algorithm_error"),+ LibsyError::Driver(_) => Cow::Borrowed("driver_error"),+ LibsyError::AlgorithmTask { .. } => Cow::Borrowed("algorithm_task_error"),+ LibsyError::MissingFinalResponse => Cow::Borrowed("missing_final_response"),+ LibsyError::AllTargetsExcluded => Cow::Borrowed("context_window_exceeded"),+ LibsyError::External { .. } => Cow::Borrowed("_OTHER"),+ }+}++fn llm_client_error_type(error: &LlmClientError) -> Cow<'static, str> {+ match error {+ LlmClientError::InvalidRequest { .. } => Cow::Borrowed("invalid_request"),+ LlmClientError::RequestTranslation(_) => Cow::Borrowed("request_translation"),+ LlmClientError::RequestEncoding(_) => Cow::Borrowed("request_encoding"),+ LlmClientError::ResponseTranslation(_) => Cow::Borrowed("response_translation"),+ LlmClientError::Configuration { .. } => Cow::Borrowed("configuration"),+ LlmClientError::Transport { .. } => Cow::Borrowed("transport"),+ LlmClientError::Timeout { .. } => Cow::Borrowed("timeout"),+ LlmClientError::ContextWindowExceeded { .. } => Cow::Borrowed("context_window_exceeded"),+ LlmClientError::UpstreamHttp { status, .. } => Cow::Owned(status.to_string()),+ LlmClientError::InvalidResponse { .. } => Cow::Borrowed("invalid_response"),+ LlmClientError::Ffi { .. } => Cow::Borrowed("ffi"),+ _ => Cow::Borrowed("_OTHER"),+ }+}++// Keep the client span alive until the response is drained, errors, or is abandoned.+struct ObservedClientStream {+ stream: LlmResponseStream,+ observer: Option<ClientStreamObserver>,+}++impl Stream for ObservedClientStream {+ type Item = std::result::Result<LlmResponseStreamEvent, LlmClientError>;++ fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Option<Self::Item>> {+ match self.stream.as_mut().poll_next(cx) {+ Poll::Ready(Some(item)) => {+ let failed = self+ .observer+ .as_mut()+ .is_some_and(|observer| observer.observe(&item));+ if failed {+ self.observer.take();+ }+ Poll::Ready(Some(item))+ }+ Poll::Ready(None) => {+ if let Some(mut observer) = self.observer.take() {+ observer.complete();+ }+ Poll::Ready(None)+ }+ Poll::Pending => Poll::Pending,+ }+ }+}++/// What the span has recorded for this stream so far.+#[derive(PartialEq)]+enum Outcome {+ /// Still streaming. Dropping the response now is a cancellation.+ Open,+ /// The model finished its message, so `outcome` is already `ok` and dropping the+ /// response is no longer a cancellation. Observation continues: OpenAI Chat sends usage+ /// in a chunk *after* the one carrying `finish_reason`, so stopping here would lose it.+ Completed,+ /// An error is recorded and terminal; nothing later is worth observing.+ Failed,+}++struct ClientStreamObserver {+ span: Span,+ outcome: Outcome,+}++impl ClientStreamObserver {+ /// Records what `item` says about the response. Returns whether the stream has failed —+ /// the only state in which the caller should stop observing.+ fn observe(+ &mut self,+ item: &std::result::Result<LlmResponseStreamEvent, LlmClientError>,+ ) -> bool {+ match item {+ Ok(event) => {+ for chunk in event.normalized() {+ self.observe_chunk(chunk);+ if self.outcome == Outcome::Failed {+ break;+ }+ }+ }+ Err(error) => {+ let error_type = llm_client_error_type(error);+ record_client_error(&self.span, &error_type, error);+ self.outcome = Outcome::Failed;+ }+ }+ self.outcome == Outcome::Failed+ }++ fn observe_chunk(&mut self, chunk: &LlmResponseChunk) {+ match chunk {+ LlmResponseChunk::MessageStart { id, model } => {+ record_optional(&self.span, "gen_ai.response.id", id.as_deref());+ record_optional(&self.span, "gen_ai.response.model", model.as_deref());+ }+ LlmResponseChunk::Usage(usage) => record_gen_ai_usage(&self.span, usage),+ LlmResponseChunk::MessageStop { reason } => {+ record_finish_reasons(&self.span, reason.iter().cloned());+ self.complete();+ }+ LlmResponseChunk::DecodeError { message } => {+ record_client_error(&self.span, "response_translation", message);+ self.outcome = Outcome::Failed;+ }+ LlmResponseChunk::StreamError { message } => {+ record_client_error(&self.span, "502", message);+ self.outcome = Outcome::Failed;+ }+ _ => {}+ }+ }++ /// Records the normal terminal outcome, unless an error already claimed it.+ fn complete(&mut self) {+ if self.outcome == Outcome::Open {+ self.span.record("outcome", "ok");+ self.outcome = Outcome::Completed;+ }+ }+}++impl Drop for ClientStreamObserver {+ fn drop(&mut self) {+ if self.outcome == Outcome::Open {+ self.span.record("outcome", "cancelled");+ }+ }+}++fn record_client_error(span: &Span, error_type: &str, error: &dyn std::fmt::Display) {+ span.record("outcome", "error");+ span.record("otel.status_code", "ERROR");+ span.record("error.type", error_type);+ span.record("error", tracing::field::display(error));+}diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rsnew file mode 100644index 000000000..efa500ab0--- /dev/null+++ b/crates/libsy-llm-client/src/run.rs@@ -0,0 +1,185 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Drive a libsy algorithm to completion, making the model calls it offloads.+//!+//! [`switchyard_libsy::Algorithm::run_stream`] is the whole libsy API: it yields a stream of+//! steps and expects its consumer to serve every offloaded model call. [`run()`] is that+//! consumer — it drives the stream with [`switchyard_libsy::drive`], hands each call to a+//! [`RoutedLlmClient`], and returns the final response with the trace of decisions.+//!+//! libsy owns the stream mechanics; what this module adds is the client call itself and the+//! `libsy.client_call` span around it.++use std::collections::HashMap;+use std::sync::Arc;++use switchyard_libsy::{+ Algorithm, CallLlmRequest, LibsyError, Result, RunObserver, algorithm_label, drive,+};+use switchyard_protocol::{Context, Decision, LlmClientError, Request, Response, RoutedLlmClient};++use crate::observability;++/// Run one request to completion, serving every offloaded model call with `client`.+///+/// Returns the final [`Response`] and the trace of [`Decision`]s the algorithm published along+/// the way. `observer`, when present, receives each completed model call and, after a+/// successful routed run, its routing overhead.+///+/// `clients` resolves each offloaded call to the client for the target the algorithm+/// selected — an algorithm may route among targets served by different providers, so this is+/// a per-call lookup, not one client for the whole run. Use+/// [`ClientRouter::single`](ClientRouter::single) when one client serves every target.+///+/// A failed *model* call is forwarded back into the algorithm, which may route around it;+/// this returns `Err` only when the run itself cannot complete.+pub async fn run(+ algorithm: Arc<dyn Algorithm>,+ clients: ClientRouter,+ ctx: Context,+ request: Request,+ observer: Option<RunObserver>,+) -> Result<(Vec<Arc<dyn Decision>>, Response)> {+ drive(algorithm, ctx, request, observer, move |call| {+ serve(clients.clone(), call)+ })+ .await+}++/// Serve one offloaded call. A failed *model* call is forwarded to the algorithm via+/// `respond`; this errors only when the promise itself could not be fulfilled. `serve` makes+/// the one provider call a routed request performs, so it gets its own `libsy.client_call`+/// span.+#[tracing::instrument(+ target = "libsy",+ name = "libsy.client_call",+ skip_all,+ fields(+ algorithm = algorithm_label(&call.get_routed().ctx),+ switchyard.algorithm = algorithm_label(&call.get_routed().ctx),+ switchyard.routing.tier = tracing::field::Empty,+ selected_model = call.get_decision().selected_model(),+ otel.kind = "client",+ otel.name = %format_args!("chat {}", call.get_decision().selected_model()),+ openinference.span.kind = "LLM",+ gen_ai.operation.name = "chat",+ gen_ai.request.model = call.get_decision().selected_model(),+ gen_ai.request.stream = tracing::field::Empty,+ gen_ai.request.temperature = tracing::field::Empty,+ gen_ai.request.top_p = tracing::field::Empty,+ gen_ai.request.top_k = tracing::field::Empty,+ gen_ai.request.max_tokens = tracing::field::Empty,+ gen_ai.request.reasoning.level = tracing::field::Empty,+ gen_ai.output.type = tracing::field::Empty,+ gen_ai.conversation.id = tracing::field::Empty,+ server.address = tracing::field::Empty,+ server.port = tracing::field::Empty,+ gen_ai.response.id = tracing::field::Empty,+ gen_ai.response.model = tracing::field::Empty,+ gen_ai.usage.input_tokens = tracing::field::Empty,+ gen_ai.usage.output_tokens = tracing::field::Empty,+ gen_ai.usage.cache_read.input_tokens = tracing::field::Empty,+ gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty,+ gen_ai.usage.reasoning.output_tokens = tracing::field::Empty,+ outcome = tracing::field::Empty,+ otel.status_code = tracing::field::Empty,+ error.type = tracing::field::Empty,+ error = tracing::field::Empty,+ )+)]+async fn serve(clients: ClientRouter, call: CallLlmRequest) -> Result<()> {+ let span = tracing::Span::current();+ observability::record_gen_ai_request(&span, &call.get_routed().request.llm_request);+ if let Some(tier) = call.get_decision().routing_tier() {+ span.record("switchyard.routing.tier", tier);+ }+ if let Some(session_id) = call+ .get_routed()+ .request+ .metadata+ .as_ref()+ .and_then(|metadata| metadata.session_id.as_deref())+ {+ span.record("gen_ai.conversation.id", session_id);+ }+ let routed = call.get_routed().clone();+ let target = routed.decision.selected_model().to_string();+ let result = match clients.route(&target) {+ Ok(client) => {+ client+ .call(routed.ctx, routed.request, routed.decision)+ .await+ }+ Err(error) => Err(error),+ }+ .map_err(|source| LibsyError::client_call(target, source));+ let result = observability::observe_client_call(result);+ call.respond(result)+}++/// Resolves a routed call's selected model to the client that serves it.+///+/// An algorithm routes among named targets; which provider each target lives on is the+/// host's concern, and two targets in one run may sit on different providers. A router owns+/// that mapping. It is *not* itself a client: it hands back a [`RoutedLlmClient`] and the+/// caller makes the call.+///+/// Cloning is cheap — the mapping is shared, so one router can serve every request.+#[derive(Clone)]+pub struct ClientRouter {+ routing: Arc<Routing>,+}++enum Routing {+ /// One client serves every model.+ Single(Arc<dyn RoutedLlmClient>),+ /// Each model is served by the client configured for it.+ ByModel(HashMap<String, Arc<dyn RoutedLlmClient>>),+}++impl ClientRouter {+ /// Build a router over `model name -> client`, for targets spread across providers.+ pub fn new(by_model: HashMap<String, Arc<dyn RoutedLlmClient>>) -> Self {+ Self {+ routing: Arc::new(Routing::ByModel(by_model)),+ }+ }++ /// A router that serves every model with one client — the single-provider case.+ ///+ /// [`TranslatingLlmClient`](crate::TranslatingLlmClient) already maps model names to+ /// backends internally and rejects ones it does not know, so enumerating them here would+ /// only duplicate that.+ pub fn single(client: Arc<dyn RoutedLlmClient>) -> Self {+ Self {+ routing: Arc::new(Routing::Single(client)),+ }+ }++ /// The client that serves `model`.+ ///+ /// Errors with [`LlmClientError::Configuration`] when the router maps models and has no+ /// entry for this one, rather than silently sending the call to another provider.+ pub fn route(+ &self,+ model: &str,+ ) -> std::result::Result<&Arc<dyn RoutedLlmClient>, LlmClientError> {+ match self.routing.as_ref() {+ Routing::Single(client) => Ok(client),+ Routing::ByModel(by_model) => {+ by_model+ .get(model)+ .ok_or_else(|| LlmClientError::Configuration {+ message: format!("no llm client is configured for model {model:?}"),+ })+ }+ }+ }+}++impl FromIterator<(String, Arc<dyn RoutedLlmClient>)> for ClientRouter {+ fn from_iter<I: IntoIterator<Item = (String, Arc<dyn RoutedLlmClient>)>>(iter: I) -> Self {+ Self::new(iter.into_iter().collect())+ }+}diff --git a/crates/libsy/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rssimilarity index 96%rename from crates/libsy/tests/observability.rsrename to crates/libsy-llm-client/tests/observability.rsindex 501352f0f..864c95f56 100644--- a/crates/libsy/tests/observability.rs+++ b/crates/libsy-llm-client/tests/observability.rs@@ -34,8 +34,9 @@ use tracing_subscriber::registry::LookupSpan;use switchyard_libsy::{Algorithm, Driver, LibsyError, LlmClassifierConfig, LlmTarget, LlmTargetSet, LlmTaskClassifier,- Step, TaskClassifierConfig,+ RoutedRequest, Step, TaskClassifierConfig,};+use switchyard_llm_client::ClientRouter;use switchyard_protocol::{Context, Decision, LlmResponse, Metadata, Request, Response, RoutedLlmClient, Usage,};@@ -474,7 +475,11 @@ impl Algorithm for SingleCallAlgo {});driver.info(ctx.clone(), decision.clone()).await?;driver- .call_llm_target(ctx, &target, request, decision)+ .call_llm(RoutedRequest {+ request,+ decision,+ ctx,+ }).await}}@@ -495,25 +500,39 @@ fn request_with_metadata(session_id: &str, correlation_id: &str) -> Request {}}-fn algo(name: &str, model: &str, client: Option<Arc<dyn RoutedLlmClient>>) -> Arc<dyn Algorithm> {+fn algo(name: &str, model: &str) -> Arc<dyn Algorithm> {Arc::new(SingleCallAlgo {name: name.to_string(),target_set: LlmTargetSet::new(vec![LlmTarget {semantic_name: model.to_string(),- llm_client: client,}]),})}+/// Drives `algorithm` to completion with `client` serving every target, the way a+/// single-provider host does.+async fn run(+ algorithm: Arc<dyn Algorithm>,+ client: Arc<dyn RoutedLlmClient>,+ request: Request,+) -> switchyard_libsy::Result<(Vec<Arc<dyn Decision>>, Response)> {+ switchyard_llm_client::run(+ algorithm,+ ClientRouter::single(client),+ Context::default(),+ request,+ None,+ )+ .await+}+fn classifier_router(judge_model: &str,efficient_model: &str,capable_model: &str,- client: Arc<dyn RoutedLlmClient>,) -> switchyard_libsy::Result<Arc<dyn Algorithm>> {let target = |name: &str| LlmTarget {semantic_name: name.to_string(),- llm_client: Some(client.clone()),};let targets = LlmTargetSet::new(vec![target(efficient_model), target(capable_model)]);Ok(Arc::new(LlmTaskClassifier::new(@@ -612,9 +631,7 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_lrequest.llm_request.output.max_output_tokens = Some(512);request.llm_request.output.response_format = Some(json!({"type": "json_schema"}));request.llm_request.reasoning.effort = Some("high".to_string());- let (trace, _response) = algo(ALGO, MODEL, Some(client))- .run(Context::default(), request)- .await?;+ let (trace, _response) = run(algo(ALGO, MODEL), client, request).await?;assert_eq!(trace.len(), 1);// Metrics: run/call counters and latency histograms keyed by algorithm,@@ -885,9 +902,7 @@ async fn streamed_usage_updates_the_client_call_span() -> switchyard_libsy::Resulet client = Arc::new(StreamingUsageClient) as Arc<dyn RoutedLlmClient>;let mut request = request_with_metadata("obs-stream-session", "obs-stream-corr");request.llm_request.stream = true;- let (_, response) = algo(ALGO, MODEL, Some(client))- .run(Context::default(), request)- .await?;+ let (_, response) = run(algo(ALGO, MODEL), client, request).await?;let LlmResponse::Stream(mut stream) = response.llm_response else {return Err(test_error("expected a streamed response"));};@@ -932,9 +947,7 @@ async fn dropped_stream_records_cancelled_outcome() -> switchyard_libsy::Result<let client = Arc::new(StreamingUsageClient) as Arc<dyn RoutedLlmClient>;let mut request = request_with_metadata("obs-cancelled-session", "obs-cancelled-corr");request.llm_request.stream = true;- let (_, response) = algo(ALGO, MODEL, Some(client))- .run(Context::default(), request)- .await?;+ let (_, response) = run(algo(ALGO, MODEL), client, request).await?;let LlmResponse::Stream(stream) = response.llm_response else {return Err(test_error("expected a streamed response"));};@@ -955,13 +968,9 @@ async fn typed_client_failure_records_semantic_error_type() {let (store, _, _, _, _) = telemetry();const ALGO: &str = "obs-timeout-algo";const MODEL: &str = "obs-timeout-model";- let result = algo(- ALGO,- MODEL,- Some(Arc::new(TimeoutClient) as Arc<dyn RoutedLlmClient>),- )- .run(- Context::default(),+ let result = run(+ algo(ALGO, MODEL),+ Arc::new(TimeoutClient),request_with_metadata("obs-timeout-session", "obs-timeout-corr"),).await;@@ -993,8 +1002,8 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::let total_errors_before =u64_gauge_value(&before, "switchyard.total_errors").unwrap_or_default();- // Client-less target: the call is offloaded and we fail it by hand.- let stream = algo(ALGO, MODEL, None).run_stream(+ // The call is offloaded and we fail it by hand, without a client.+ let stream = algo(ALGO, MODEL).run_stream(Context::default(),request_with_metadata("obs-session-2", "obs-corr-2"),None,@@ -1117,9 +1126,9 @@ async fn classifier_metrics_count_only_the_final_routed_call() -> switchyard_libclassifier_delay: Duration::from_millis(60),routed_delay: Duration::from_millis(200),}) as Arc<dyn RoutedLlmClient>;- let router = classifier_router("classifier", "weak", "strong", client)?;+ let router = classifier_router("classifier", "weak", "strong")?;- let (trace, _response) = router.run(Context::default(), classifier_request()).await?;+ let (trace, _response) = run(router, client, classifier_request()).await?;assert_eq!(trace.last().and_then(|decision| decision.routing_tier()),@@ -1210,9 +1219,12 @@ async fn classifier_fail_open_records_each_failure_stage() -> switchyard_libsy::for (judge_model, outcome, expected_reason) in cases {let client = Arc::new(JudgeClient { outcome }) as Arc<dyn RoutedLlmClient>;- classifier_router(judge_model, "fo-weak", "fo-strong", client)?- .run(Context::default(), classifier_request())- .await?;+ run(+ classifier_router(judge_model, "fo-weak", "fo-strong")?,+ client,+ classifier_request(),+ )+ .await?;let snapshots = flushed_metrics(exporter, provider);match expected_reason {diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.tomlindex bea219160..f97d06a80 100644--- a/crates/libsy/Cargo.toml+++ b/crates/libsy/Cargo.toml@@ -33,11 +33,10 @@ thiserror.workspace = truetokio.workspace = truetokio-stream = "0.1"tracing.workspace = true-tracing-opentelemetry = "0.33"+tracing-opentelemetry.workspace = true[dev-dependencies]# SDK + in-memory exporter to assert what the observability layer records.opentelemetry_sdk = { version = "0.32", features = ["metrics", "testing", "trace"] }-switchyard-llm-client.workspace = truetokio.workspace = true-tracing-subscriber = "0.3"+tracing-subscriber.workspace = truediff --git a/crates/libsy/README.md b/crates/libsy/README.mdindex 41907ecfe..9f0f95aa4 100644--- a/crates/libsy/README.md+++ b/crates/libsy/README.md@@ -2,9 +2,9 @@Provider-neutral orchestration for multi-LLM optimization. A libsy[`Algorithm`] decides which model targets to call, in what order, and how to-combine their results. It can use target-owned clients or hand each call back to-the host, allowing it to embed in proxies, gateways, and agent runtimes without-owning an HTTP stack.+combine their results. It hands every call back to the host rather than making+it, allowing it to embed in proxies, gateways, and agent runtimes without owning+an HTTP stack.## Setup@@ -30,12 +30,15 @@ tokio = { version = "1", features = ["macros", "rt"] }## How it fits together-[`LlmTarget`] pairs a semantic routing name with an optional-[`RoutedLlmClient`](switchyard_protocol::RoutedLlmClient). An [`Algorithm`]-selects targets and records [`Decision`](switchyard_protocol::Decision)s. Use-[`Algorithm::run`] with target-owned clients, or [`Algorithm::run_stream`] when-the host owns model transport. The provider-neutral [`Request`], [`Response`],-[`Usage`], and [`LlmResponse`] contracts come from `switchyard-protocol`.+[`LlmTarget`] names a routing destination. An [`Algorithm`] selects targets and+records [`Decision`](switchyard_protocol::Decision)s, offloading every model call+to its caller: [`Algorithm::run_stream`] yields a [`Step`] stream whose+[`Step::CallLlm`] items the host serves over its own transport. libsy makes no+network calls itself — `switchyard-llm-client`'s `run` is a ready-made consumer+that drives the stream and performs the calls over HTTP.++The provider-neutral [`Request`], [`Response`], [`Usage`], and [`LlmResponse`]+contracts come from `switchyard-protocol`.[`Request`]: switchyard_protocol::Request[`Response`]: switchyard_protocol::Responsediff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rsindex 96a7e5093..331943354 100644--- a/crates/libsy/src/algorithms/fall_through.rs+++ b/crates/libsy/src/algorithms/fall_through.rs@@ -463,9 +463,9 @@ mod tests {use crate::core::classifier::Classification;use crate::{AffinityRouter, SystemPromptProcessor, TargetPrompts};+ use crate::core::testing::{Serve, drive, echo, reply};use switchyard_protocol::{- LlmClientError, LlmRequest, LlmResponse, Message, Metadata, Role, RoutedLlmClient,- completion_text, text_request, text_response,+ LlmClientError, LlmRequest, Message, Metadata, Role, completion_text, text_request,};#[derive(Debug, thiserror::Error)]@@ -478,47 +478,15 @@ mod tests {// --- fixtures ----------------------------------------------------------------------- /// A client that echoes the routed model name back as the completion.- struct EchoClient;-- #[async_trait]- impl RoutedLlmClient for EchoClient {- async fn call(- &self,- _ctx: Context,- _request: Request,- decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {- Ok(Response {- llm_response: LlmResponse::Agg(text_response(- None,- decision.selected_model().to_string(),- )),- metadata: None,- })- }- }-- /// A client that captures the request it was handed, so a test can assert on what- /// actually reached the model.- struct CapturingClient(Arc<parking_lot::Mutex<Option<Request>>>);-- #[async_trait]- impl RoutedLlmClient for CapturingClient {- async fn call(- &self,- _ctx: Context,- request: Request,- decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {- *self.0.lock() = Some(request);- Ok(Response {- llm_response: LlmResponse::Agg(text_response(- None,- decision.selected_model().to_string(),- )),- metadata: None,- })+ /// Echoes the routed model name, capturing the request it was handed so a test can+ /// assert on what actually reached the model.+ fn capturing(into: Arc<Mutex<Option<Request>>>) -> impl Serve {+ move |decision: Arc<dyn Decision>, request: Request| {+ let into = Arc::clone(&into);+ async move {+ *into.lock() = Some(request);+ Ok(reply(decision.selected_model()))+ }}}@@ -536,35 +504,32 @@ mod tests {/// Captures the prompt-bearing request that reached the selected target.#[derive(Default)]- struct RecordingPromptClient(Mutex<Option<RecordedCall>>);-- #[async_trait]- impl RoutedLlmClient for RecordingPromptClient {- async fn call(- &self,- _ctx: Context,- request: Request,- decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {- *self.0.lock() = Some(RecordedCall {- target: decision.selected_model().to_string(),- messages: request- .llm_request- .messages- .iter()- .filter_map(|message| message.text_content("|"))- .collect(),- instructions: request- .llm_request- .instructions- .iter()- .filter_map(|block| block.content.iter().find_map(text_of))- .collect(),- });- Ok(Response {- llm_response: LlmResponse::Agg(text_response(None, decision.selected_model())),- metadata: None,- })+ struct PromptRecorder(Mutex<Option<RecordedCall>>);++ impl PromptRecorder {+ fn serve(self: &Arc<Self>) -> impl Serve {+ let recorder = Arc::clone(self);+ move |decision: Arc<dyn Decision>, request: Request| {+ let recorder = Arc::clone(&recorder);+ async move {+ *recorder.0.lock() = Some(RecordedCall {+ target: decision.selected_model().to_string(),+ messages: request+ .llm_request+ .messages+ .iter()+ .filter_map(|message| message.text_content("|"))+ .collect(),+ instructions: request+ .llm_request+ .instructions+ .iter()+ .filter_map(|block| block.content.iter().find_map(text_of))+ .collect(),+ });+ Ok(reply(decision.selected_model()))+ }+ }}}@@ -575,26 +540,12 @@ mod tests {}}- /// A target set whose targets all serve via [`EchoClient`].fn target_set(names: &[&str]) -> LlmTargetSet {LlmTargetSet::new(names.iter().map(|name| LlmTarget {semantic_name: name.to_string(),- llm_client: Some(Arc::new(EchoClient) as Arc<dyn RoutedLlmClient>),- })- .collect(),- )- }-- fn prompt_targets(client: &Arc<RecordingPromptClient>, names: &[&str]) -> LlmTargetSet {- LlmTargetSet::new(- names- .iter()- .map(|name| LlmTarget {- semantic_name: (*name).to_string(),- llm_client: Some(client.clone() as Arc<dyn RoutedLlmClient>),}).collect(),)@@ -608,20 +559,21 @@ mod tests {/// Routes one turn on a prompt test cascade and returns the recorded model call.async fn routed_prompt_call(- client: &Arc<RecordingPromptClient>,+ recorder: &Arc<PromptRecorder>,router: FallThrough,) -> Result<RecordedCall> {- Arc::new(router)- .run(- Context::default(),- Request {- llm_request: text_request(Some("auto".to_string()), "fix the build"),- raw_request: None,- metadata: None,- },- )- .await?;- let call = client.0.lock().take();+ drive(+ Arc::new(router),+ Context::default(),+ Request {+ llm_request: text_request(Some("auto".to_string()), "fix the build"),+ raw_request: None,+ metadata: None,+ },+ recorder.serve(),+ )+ .await?;+ let call = recorder.0.lock().take();match call {Some(call) => Ok(call),None => panic!("the model was never called"),@@ -629,12 +581,8 @@ mod tests {}/// A prompt cascade that always routes to `target`.- fn prompt_router(- client: &Arc<RecordingPromptClient>,- target: &str,- prompts: TargetPrompts,- ) -> FallThrough {- FallThrough::new(prompt_targets(client, &["capable", "efficient"]))+ fn prompt_router(target: &str, prompts: TargetPrompts) -> FallThrough {+ FallThrough::new(target_set(&["capable", "efficient"])).with_processor(Arc::new(SystemPromptProcessor::new(prompts))).with_classifier(Arc::new(DefaultTarget::new(target)))}@@ -695,11 +643,12 @@ mod tests {async fn run_request<S>(router: &Arc<FallThrough<S>>,request: Request,+ serve: impl Serve,) -> Result<(String, Vec<Arc<dyn Decision>>)>whereS: Default + Send + 'static,{- let (trace, response) = router.clone().run(Context::default(), request).await?;+ let (trace, response) = drive(router.clone(), Context::default(), request, serve).await?;let text = response.llm_response.into_agg()@@ -710,16 +659,71 @@ mod tests {}/// Drives a shared router through one turn in the default test session.- async fn run_turn<S>(router: &Arc<FallThrough<S>>) -> Result<(String, Vec<Arc<dyn Decision>>)>+ async fn run_turn<S>(+ router: &Arc<FallThrough<S>>,+ serve: impl Serve,+ ) -> Result<(String, Vec<Arc<dyn Decision>>)>whereS: Default + Send + 'static,{- run_request(router, request()).await+ run_request(router, request(), serve).await+ }++ /// Drives a fresh router through one turn with a specific `serve`.+ async fn run_with(+ router: FallThrough,+ serve: impl Serve,+ ) -> Result<(String, Vec<Arc<dyn Decision>>)> {+ run_turn(&Arc::new(router), serve).await+ }++ /// Discards the recorded calls; the overflow tests that don't assert on them.+ fn overflows(targets: &'static [&'static str]) -> impl Serve {+ overflowing(targets, Arc::new(Mutex::new(Vec::new())))}- /// Drives a fresh router through one turn.- async fn run(router: FallThrough) -> Result<(String, Vec<Arc<dyn Decision>>)> {- run_turn(&Arc::new(router)).await+ /// Rejects the named `overflowing` targets with a context-window error and echoes for+ /// the rest, so the retry path can be driven. Every call is recorded in `calls`.+ fn overflowing(+ overflowing: &'static [&'static str],+ calls: Arc<Mutex<Vec<String>>>,+ ) -> impl Serve {+ move |decision: Arc<dyn Decision>, _request: Request| {+ let calls = Arc::clone(&calls);+ async move {+ let model = decision.selected_model().to_string();+ calls.lock().push(model.clone());+ if overflowing.contains(&model.as_str()) {+ return Err(LlmClientError::ContextWindowExceeded {+ model,+ message: "prompt is too long".to_string(),+ });+ }+ Ok(reply(model))+ }+ }+ }++ /// Rejects the named `unavailable` targets with a 503 and echoes for the rest.+ /// Every call is recorded in `calls`.+ fn unavailable(+ unavailable: &'static [&'static str],+ calls: Arc<Mutex<Vec<String>>>,+ ) -> impl Serve {+ move |decision: Arc<dyn Decision>, _request: Request| {+ let calls = Arc::clone(&calls);+ async move {+ let model = decision.selected_model().to_string();+ calls.lock().push(model.clone());+ if unavailable.contains(&model.as_str()) {+ return Err(LlmClientError::UpstreamHttp {+ status: 503,+ body: "unavailable".to_string(),+ });+ }+ Ok(reply(model))+ }+ }}// --- tests -------------------------------------------------------------------------@@ -727,10 +731,9 @@ mod tests {#[tokio::test]async fn each_target_gets_its_own_prompt() -> Result<()> {for (target, expected) in [("capable", CAPABLE_PROMPT), ("efficient", EFFICIENT_PROMPT)] {- let client = Arc::new(RecordingPromptClient::default());+ let recorder = Arc::new(PromptRecorder::default());let call =- routed_prompt_call(&client, prompt_router(&client, target, target_prompts()))- .await?;+ routed_prompt_call(&recorder, prompt_router(target, target_prompts())).await?;assert_eq!(call.target, target);assert_eq!(call.instructions, vec![expected.to_string()]);}@@ -739,11 +742,10 @@ mod tests {#[tokio::test]async fn a_target_with_no_prompt_is_left_untouched() -> Result<()> {- let client = Arc::new(RecordingPromptClient::default());+ let recorder = Arc::new(PromptRecorder::default());let only_capable = TargetPrompts::default().with("capable", CAPABLE_PROMPT);- let call =- routed_prompt_call(&client, prompt_router(&client, "efficient", only_capable)).await?;+ let call = routed_prompt_call(&recorder, prompt_router("efficient", only_capable)).await?;assert!(call.instructions.is_empty(),@@ -771,13 +773,13 @@ mod tests {}}- let client = Arc::new(RecordingPromptClient::default());- let router = FallThrough::new(prompt_targets(&client, &["capable", "efficient"]))+ let recorder = Arc::new(PromptRecorder::default());+ let router = FallThrough::new(target_set(&["capable", "efficient"])).with_processor(Arc::new(SystemPromptProcessor::new(target_prompts()))).with_classifier(Arc::new(Abstains)).with_classifier(Arc::new(DefaultTarget::new("capable")));- let call = routed_prompt_call(&client, router).await?;+ let call = routed_prompt_call(&recorder, router).await?;assert_eq!(call.target, "capable");assert_eq!(call.instructions, vec![CAPABLE_PROMPT.to_string()]);@@ -800,137 +802,28 @@ mod tests {}}- let client = Arc::new(RecordingPromptClient::default());- let router = FallThrough::new(prompt_targets(&client, &["capable", "efficient"]))+ let recorder = Arc::new(PromptRecorder::default());+ let router = FallThrough::new(target_set(&["capable", "efficient"])).with_processor(Arc::new(Noting)).with_classifier(Arc::new(DefaultTarget::new("capable")));- let call = routed_prompt_call(&client, router).await?;+ let call = routed_prompt_call(&recorder, router).await?;assert_eq!(call.messages, vec![format!("fix the build|{NOTE}")]);assert!(call.instructions.is_empty(), "a note is not an instruction");Ok(())}- /// Overflows for the named targets and echoes for the rest, recording every call.- struct OverflowClient {- overflowing: Vec<&'static str>,- calls: Option<Arc<Mutex<Vec<String>>>>,- }-- /// Returns 503 for `weak` and records every routed call.- struct UnavailableClient(Arc<Mutex<Vec<String>>>);-- #[async_trait]- impl RoutedLlmClient for UnavailableClient {- async fn call(- &self,- _ctx: Context,- _request: Request,- decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, LlmClientError> {- let model = decision.selected_model().to_string();- self.0.lock().push(model.clone());- if model == "weak" {- return Err(LlmClientError::UpstreamHttp {- status: 503,- body: "unavailable".to_string(),- });- }- Ok(Response {- llm_response: LlmResponse::Agg(text_response(None, model)),- metadata: None,- })- }- }-- fn unavailable_targets(calls: Arc<Mutex<Vec<String>>>) -> LlmTargetSet {- let client: Arc<dyn RoutedLlmClient> = Arc::new(UnavailableClient(calls));- LlmTargetSet::new(- ["weak", "strong"]- .into_iter()- .map(|name| LlmTarget {- semantic_name: name.to_string(),- llm_client: Some(Arc::clone(&client)),- })- .collect(),- )- }-- #[async_trait]- impl RoutedLlmClient for OverflowClient {- async fn call(- &self,- _ctx: Context,- _request: Request,- decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, LlmClientError> {- let model = decision.selected_model().to_string();- if let Some(calls) = &self.calls {- calls.lock().push(model.clone());- }- if self.overflowing.contains(&model.as_str()) {- return Err(LlmClientError::ContextWindowExceeded {- model,- message: "prompt is too long".to_string(),- });- }- Ok(Response {- llm_response: LlmResponse::Agg(text_response(None, model)),- metadata: None,- })- }- }-- /// `target_set`, but the named `overflowing` targets reject every call with a- /// context-window error so the retry path can be driven.- fn target_set_with_overflow(names: &[&str], overflowing: &[&'static str]) -> LlmTargetSet {- LlmTargetSet::new(- names- .iter()- .map(|name| LlmTarget {- semantic_name: name.to_string(),- llm_client: Some(Arc::new(OverflowClient {- overflowing: overflowing.to_vec(),- calls: None,- }) as Arc<dyn RoutedLlmClient>),- })- .collect(),- )- }-- fn counting_overflow_targets(- names: &[&str],- overflowing: &[&'static str],- calls: Arc<Mutex<Vec<String>>>,- ) -> LlmTargetSet {- LlmTargetSet::new(- names- .iter()- .map(|name| LlmTarget {- semantic_name: name.to_string(),- llm_client: Some(Arc::new(OverflowClient {- overflowing: overflowing.to_vec(),- calls: Some(calls.clone()),- }) as Arc<dyn RoutedLlmClient>),- })- .collect(),- )- }-#[tokio::test]async fn a_target_that_overflowed_is_skipped_for_the_rest_of_the_session() -> Result<()> {let calls = Arc::new(Mutex::new(Vec::new()));let router = Arc::new(- FallThrough::<()>::new(counting_overflow_targets(- &["weak", "strong"],- &["weak"],- calls.clone(),- ))- .with_classifier(fixed(vec![score("weak", 0.9)])),+ FallThrough::<()>::new(target_set(&["weak", "strong"]))+ .with_classifier(fixed(vec![score("weak", 0.9)])),);for _ in 0..3 {- assert_eq!(run_turn(&router).await?.0, "strong");+ let serve = overflowing(&["weak"], calls.clone());+ assert_eq!(run_turn(&router, serve).await?.0, "strong");}assert_eq!(calls.lock().iter().filter(|m| *m == "weak").count(), 1);Ok(())@@ -940,20 +833,16 @@ mod tests {async fn a_different_session_starts_with_an_empty_eviction_set() -> Result<()> {let calls = Arc::new(Mutex::new(Vec::new()));let router = Arc::new(- FallThrough::<()>::new(counting_overflow_targets(- &["weak", "strong"],- &["weak"],- calls.clone(),- ))- .with_classifier(fixed(vec![score("weak", 0.9)])),+ FallThrough::<()>::new(target_set(&["weak", "strong"]))+ .with_classifier(fixed(vec![score("weak", 0.9)])),);- run_turn(&router).await?;+ run_turn(&router, overflowing(&["weak"], calls.clone())).await?;let mut other = request();other.metadata = Some(Metadata {session_id: Some("session-2".to_string()),..Metadata::default()});- run_request(&router, other).await?;+ run_request(&router, other, overflowing(&["weak"], calls.clone())).await?;assert_eq!(calls.lock().iter().filter(|m| *m == "weak").count(), 2);Ok(())}@@ -962,17 +851,13 @@ mod tests {async fn second_turn_after_full_exhaustion_still_reaches_upstream() -> Result<()> {let calls = Arc::new(Mutex::new(Vec::new()));let router = Arc::new(- FallThrough::<()>::new(counting_overflow_targets(- &["weak", "strong"],- &["weak", "strong"],- calls.clone(),- ))- .with_classifier(fixed(vec![score("weak", 0.9)])),+ FallThrough::<()>::new(target_set(&["weak", "strong"]))+ .with_classifier(fixed(vec![score("weak", 0.9)])),);- let first = run_turn(&router).await;+ let first = run_turn(&router, overflowing(&["weak", "strong"], calls.clone())).await;assert!(first.is_err());calls.lock().clear();- match run_turn(&router).await {+ match run_turn(&router, overflowing(&["weak", "strong"], calls.clone())).await {Err(LibsyError::ClientCall { .. }) => {}Err(other) => panic!("turn 2 gave {other:?}, calls={:?}", calls.lock()),Ok(_) => panic!("expected an error"),@@ -982,10 +867,9 @@ mod tests {#[tokio::test]async fn an_overflowing_target_is_retried_on_one_that_fits() -> Result<()> {- let router =- FallThrough::<()>::new(target_set_with_overflow(&["weak", "strong"], &["weak"]))- .with_classifier(fixed(vec![score("weak", 0.9)]));- let (model, _) = run(router).await?;+ let router = FallThrough::<()>::new(target_set(&["weak", "strong"]))+ .with_classifier(fixed(vec![score("weak", 0.9)]));+ let (model, _) = run_with(router, overflows(&["weak"])).await?;assert_eq!(model, "strong");Ok(())}@@ -995,14 +879,13 @@ mod tests {let calls = Arc::new(Mutex::new(Vec::new()));let affinity = Arc::new(AffinityRouter::new());let router = Arc::new(- FallThrough::<()>::new(unavailable_targets(Arc::clone(&calls)))+ FallThrough::<()>::new(target_set(&["weak", "strong"])).with_processor(affinity.clone()).with_classifier(affinity).with_classifier(Arc::new(DefaultTarget::new("weak"))),);-for _ in 0..2 {- let (model, trace) = run_turn(&router).await?;+ let (model, trace) = run_turn(&router, unavailable(&["weak"], calls.clone())).await?;assert_eq!(model, "strong");assert_eq!(trace.last().and_then(|decision| decision.fallback_reason()),@@ -1015,12 +898,9 @@ mod tests {#[tokio::test]async fn overflowing_targets_are_retried_until_one_fits() -> Result<()> {- let router = FallThrough::<()>::new(target_set_with_overflow(- &["weak", "mid", "strong"],- &["weak", "mid"],- ))- .with_classifier(fixed(vec![score("weak", 0.9)]));- let (model, _) = run(router).await?;+ let router = FallThrough::<()>::new(target_set(&["weak", "mid", "strong"]))+ .with_classifier(fixed(vec![score("weak", 0.9)]));+ let (model, _) = run_with(router, overflows(&["weak", "mid"])).await?;assert_eq!(model, "strong");Ok(())}@@ -1028,12 +908,9 @@ mod tests {#[tokio::test]async fn exhausting_every_target_surfaces_the_client_overflow() -> Result<()> {// Only the client error maps to a 400 upstream, so it must survive exhaustion.- let router = FallThrough::<()>::new(target_set_with_overflow(- &["weak", "strong"],- &["weak", "strong"],- ))- .with_classifier(fixed(vec![score("weak", 0.9)]));- match run(router).await {+ let router = FallThrough::<()>::new(target_set(&["weak", "strong"]))+ .with_classifier(fixed(vec![score("weak", 0.9)]));+ match run_with(router, overflows(&["weak", "strong"])).await {Ok(_) => panic!("expected an overflow error, got a response"),Err(LibsyError::ClientCall {source: LlmClientError::ContextWindowExceeded { .. },@@ -1062,11 +939,10 @@ mod tests {}let seen = Arc::new(Mutex::new(Vec::new()));- let router =- FallThrough::<()>::new(target_set_with_overflow(&["weak", "strong"], &["weak"]))- .with_classifier(fixed(vec![score("weak", 0.9)]))- .with_processor(Arc::new(CountingProcessor(seen.clone())));- let (model, _) = run(router).await?;+ let router = FallThrough::<()>::new(target_set(&["weak", "strong"]))+ .with_classifier(fixed(vec![score("weak", 0.9)]))+ .with_processor(Arc::new(CountingProcessor(seen.clone())));+ let (model, _) = run_with(router, overflows(&["weak"])).await?;assert_eq!(model, "strong");assert_eq!(seen.lock().iter().filter(|e| **e == "request").count(), 1);assert_eq!(seen.lock().iter().filter(|e| **e == "decision").count(), 1);@@ -1082,7 +958,7 @@ mod tests {);let mut ctx = Context::default();ctx.exclude_target("weak");- let (trace, response) = router.run(ctx, request()).await?;+ let (trace, response) = drive(router, ctx, request(), echo()).await?;let text = response.llm_response.into_agg()@@ -1104,7 +980,7 @@ mod tests {async fn argmax_picks_the_highest_confidence_target() -> Result<()> {let router = FallThrough::<()>::new(target_set(&["strong", "weak"])).with_classifier(fixed(vec![score("weak", 0.2), score("strong", 0.9)]));- let (model, trace) = run(router).await?;+ let (model, trace) = run_with(router, echo()).await?;assert_eq!(model, "strong");assert_eq!(trace.len(), 1);assert_eq!(trace[0].selected_model(), "strong");@@ -1117,7 +993,7 @@ mod tests {let router = FallThrough::<()>::new(target_set(&["strong", "weak"])).with_classifier(fixed(vec![])).with_classifier(fixed(vec![score("weak", 1.0)]));- let (model, _) = run(router).await?;+ let (model, _) = run_with(router, echo()).await?;assert_eq!(model, "weak");Ok(())}@@ -1128,7 +1004,7 @@ mod tests {let router = FallThrough::<()>::new(target_set(&["strong", "weak"])).with_classifier(fixed(vec![score("strong", 0.6)])).with_classifier(fixed(vec![score("weak", 1.0)]));- let (model, _) = run(router).await?;+ let (model, _) = run_with(router, echo()).await?;assert_eq!(model, "strong");Ok(())}@@ -1137,7 +1013,7 @@ mod tests {async fn all_abstaining_is_an_error() -> Result<()> {let router =FallThrough::<()>::new(target_set(&["strong", "weak"])).with_classifier(fixed(vec![]));- let error = run(router)+ let error = run_with(router, echo()).await.err().ok_or_else(|| test_error("expected classifiers to abstain"))?;@@ -1171,7 +1047,7 @@ mod tests {let router = FallThrough::<()>::new(target_set(&["strong", "weak"])).with_classifier(Arc::new(NeedsDriver));- let (model, _) = run(router).await?;+ let (model, _) = run_with(router, echo()).await?;assert_eq!(model, "strong");Ok(())}@@ -1201,7 +1077,7 @@ mod tests {let router = FallThrough::<()>::new(target_set(&["strong", "weak"])).with_processor(Arc::new(RecordingProcessor(seen.clone()))).with_classifier(fixed(vec![score("strong", 1.0)]));- run(router).await?;+ run_with(router, echo()).await?;assert_eq!(*seen.lock(), vec!["request", "decision"]);Ok(())@@ -1255,16 +1131,13 @@ mod tests {let seen_by_classifier = Arc::new(Mutex::new(Vec::new()));let seen_by_model = Arc::new(Mutex::new(None));- let targets = LlmTargetSet::new(vec![LlmTarget {- semantic_name: "strong".to_string(),- llm_client: Some(Arc::new(CapturingClient(seen_by_model.clone()))),- }]);+ let targets = target_set(&["strong"]);let router = FallThrough::new(targets).with_processor(Arc::new(Appender("first"))).with_processor(Arc::new(Appender("second"))).with_classifier(Arc::new(TrailClassifier(seen_by_classifier.clone())));- run_turn(&Arc::new(router)).await?;+ run_turn(&Arc::new(router), capturing(seen_by_model.clone())).await?;// The classifier saw both processors' edits, in chain order, on top of the original.assert_eq!(*seen_by_classifier.lock(), vec!["hi", "first", "second"]);@@ -1326,8 +1199,8 @@ mod tests {.with_classifier(Arc::new(ThresholdClassifier)),);- let (turn1, _) = run_turn(&router).await?;- let (turn2, _) = run_turn(&router).await?;+ let (turn1, _) = run_turn(&router, echo()).await?;+ let (turn2, _) = run_turn(&router, echo()).await?;let final_request = Request {metadata: Some(Metadata {session_id: Some("session-1".to_string()),@@ -1336,8 +1209,8 @@ mod tests {}),..request()};- let (final_turn, _) = run_request(&router, final_request).await?;- let (restarted_session, _) = run_turn(&router).await?;+ let (final_turn, _) = run_request(&router, final_request, echo()).await?;+ let (restarted_session, _) = run_turn(&router, echo()).await?;let (second_session, _) = run_request(&router,Request {@@ -1347,14 +1220,15 @@ mod tests {}),..request()},+ echo(),).await?;let anonymous = Request {metadata: None,..request()};- let (anonymous1, _) = run_request(&router, anonymous.clone()).await?;- let (anonymous2, _) = run_request(&router, anonymous).await?;+ let (anonymous1, _) = run_request(&router, anonymous.clone(), echo()).await?;+ let (anonymous2, _) = run_request(&router, anonymous, echo()).await?;assert_eq!(turn1, "weak");assert_eq!(turn2, "strong");@@ -1378,7 +1252,7 @@ mod tests {..request()};- let result = router.clone().run(Context::default(), final_request).await;+ let result = drive(router.clone(), Context::default(), final_request, echo()).await;assert!(matches!(result, Err(LibsyError::AlgorithmError { .. })));let states = routerdiff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rsindex 95f50d312..d75748d14 100644--- a/crates/libsy/src/algorithms/llm_class.rs+++ b/crates/libsy/src/algorithms/llm_class.rs@@ -21,7 +21,7 @@ use super::util::llm_judge::{SerdeDecoder, StructuredJudge,};use super::util::target_selector::TargetSelectorPolicy;-use crate::core::algorithm::{Algorithm, Driver, LlmTarget, LlmTargetSet};+use crate::core::algorithm::{Algorithm, Driver, LlmTarget, LlmTargetSet, RoutedRequest};use crate::core::classifier::{Classification, Classifier, Score};use crate::core::state::{State, StateValue};use crate::{LibsyError, Result};@@ -519,15 +519,14 @@ impl Classifier<State> for EscalationClassifier {// `(decisive(capable), None)` tells FallThrough::execute to call// call_llm_with_fallback with the capable target instead of surfacing the error.let efficient_response = match driver- .call_llm_target(- Context::default(),- &self.efficient,- request.clone(),- Arc::new(SimpleDecision {+ .call_llm(RoutedRequest {+ request: request.clone(),+ decision: Arc::new(SimpleDecision {selected_model: self.efficient.semantic_name.clone(),reasoning: Some("escalation classifier: efficient tier".into()),}),- )+ ctx: Context::default(),+ }).await{Ok(r) => r,@@ -960,8 +959,8 @@ mod tests {};use crate::algorithms::util::llm_judge::Judge;- use crate::core::algorithm::Algorithm;- use switchyard_protocol::{Context, LlmResponse, Response, RoutedLlmClient};+ use crate::core::testing::{Serve, drive, reply};+ use switchyard_protocol::{Context, LlmResponse, Response};const TEST_THRESHOLD: f64 = 0.5;@@ -1002,14 +1001,16 @@ mod tests {})}+ /// Records what each target received; answers the judge with a supported verdict and+ /// every other target with a plain completion.#[derive(Default)]- struct PerRequestClient {+ struct Recorder {calls: Mutex<Vec<String>>,judge_max_output_tokens: Mutex<Vec<Option<u64>>>,judge_system_prompts: Mutex<Vec<String>>,}- impl PerRequestClient {+ impl Recorder {fn calls(&self) -> Vec<String> {self.calls.lock().clone()}@@ -1021,58 +1022,50 @@ mod tests {fn judge_system_prompts(&self) -> Vec<String> {self.judge_system_prompts.lock().clone()}- }- #[async_trait]- impl RoutedLlmClient for PerRequestClient {- async fn call(- &self,- _ctx: Context,- request: Request,- decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, LlmClientError> {- let model = decision.selected_model().to_string();- self.calls.lock().push(model.clone());- let completion = if model == "judge" {- self.judge_max_output_tokens- .lock()- .push(request.llm_request.output.max_output_tokens);- self.judge_system_prompts.lock().extend(- request- .llm_request- .instructions- .first()- .and_then(|instruction| {- instruction.content.iter().find_map(|b| {- if let ContentBlock::Text { text } = b {- Some(text.clone())- } else {- None- }- })- }),- );- r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#.to_string()- } else {- format!("answer from {model}")- };- Ok(Response {- llm_response: LlmResponse::Agg(text_response(None, completion)),- metadata: request.metadata,- })+ fn serve(self: &Arc<Self>) -> impl Serve {+ let recorder = Arc::clone(self);+ move |decision: Arc<dyn Decision>, request: Request| {+ let recorder = Arc::clone(&recorder);+ async move {+ let model = decision.selected_model().to_string();+ recorder.calls.lock().push(model.clone());+ let completion = if model == "judge" {+ recorder+ .judge_max_output_tokens+ .lock()+ .push(request.llm_request.output.max_output_tokens);+ recorder.judge_system_prompts.lock().extend(+ request+ .llm_request+ .instructions+ .first()+ .and_then(|instruction| {+ instruction.content.iter().find_map(|b| {+ if let ContentBlock::Text { text } = b {+ Some(text.clone())+ } else {+ None+ }+ })+ }),+ );+ r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#.to_string()+ } else {+ format!("answer from {model}")+ };+ Ok(Response {+ llm_response: LlmResponse::Agg(text_response(None, completion)),+ metadata: request.metadata,+ })+ }+ }}}- struct UnreachableJudgeClient;-- #[async_trait]- impl RoutedLlmClient for UnreachableJudgeClient {- async fn call(- &self,- _ctx: Context,- request: Request,- decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, LlmClientError> {+ /// The judge times out; every other target answers normally.+ fn unreachable_judge() -> impl Serve {+ |decision: Arc<dyn Decision>, request: Request| async move {let model = decision.selected_model().to_string();if model == "judge" {return Err(LlmClientError::Timeout {@@ -1086,10 +1079,9 @@ mod tests {}}- fn router(client: Arc<dyn RoutedLlmClient>) -> Result<Arc<LlmTaskClassifier>> {+ fn router() -> Result<Arc<LlmTaskClassifier>> {let target = |name: &str| LlmTarget {semantic_name: name.to_string(),- llm_client: Some(client.clone()),};Ok(Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {@@ -1134,9 +1126,15 @@ mod tests {#[tokio::test]async fn an_unreachable_judge_routes_capable_instead_of_failing_the_request() -> Result<()> {- let router = router(Arc::new(UnreachableJudgeClient))?;+ let router = router()?;- let (trace, response) = router.run(Context::default(), classify_request()).await?;+ let (trace, response) = drive(+ router,+ Context::default(),+ classify_request(),+ unreachable_judge(),+ )+ .await?;assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));assert_eq!(@@ -1148,15 +1146,27 @@ mod tests {#[tokio::test]async fn classifier_judges_each_request_without_affinity() -> Result<()> {- let client = Arc::new(PerRequestClient::default());- let router = router(client.clone())?;+ let recorder = Arc::new(Recorder::default());+ let router = router()?;let request = classify_request;- router.clone().run(Context::default(), request()).await?;- router.clone().run(Context::default(), request()).await?;+ drive(+ router.clone(),+ Context::default(),+ request(),+ recorder.serve(),+ )+ .await?;+ drive(+ router.clone(),+ Context::default(),+ request(),+ recorder.serve(),+ )+ .await?;assert_eq!(- client.calls(),+ recorder.calls(),vec!["judge", "efficient", "judge", "efficient"]);Ok(())@@ -1164,10 +1174,9 @@ mod tests {#[tokio::test]async fn classifier_config_sets_the_judge_completion_cap() -> Result<()> {- let client = Arc::new(PerRequestClient::default());+ let recorder = Arc::new(Recorder::default());let target = |name: &str| LlmTarget {semantic_name: name.to_string(),- llm_client: Some(client.clone()),};let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {judge_target: target("judge"),@@ -1179,18 +1188,23 @@ mod tests {},})?);- router.run(Context::default(), classify_request()).await?;+ drive(+ router,+ Context::default(),+ classify_request(),+ recorder.serve(),+ )+ .await?;- assert_eq!(client.judge_max_output_tokens(), vec![Some(512)]);+ assert_eq!(recorder.judge_max_output_tokens(), vec![Some(512)]);Ok(())}#[tokio::test]async fn classifier_config_overrides_the_packaged_prompt() -> Result<()> {- let client = Arc::new(PerRequestClient::default());+ let recorder = Arc::new(Recorder::default());let target = |name: &str| LlmTarget {semantic_name: name.to_string(),- llm_client: Some(client.clone()),};let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {judge_target: target("judge"),@@ -1203,9 +1217,15 @@ mod tests {},})?);- router.run(Context::default(), classify_request()).await?;+ drive(+ router,+ Context::default(),+ classify_request(),+ recorder.serve(),+ )+ .await?;- let prompts = client.judge_system_prompts();+ let prompts = recorder.judge_system_prompts();assert_eq!(prompts.len(), 1);assert_eq!(prompts[0], "Custom capability rubric.");Ok(())@@ -1213,10 +1233,9 @@ mod tests {#[tokio::test]async fn classifier_config_enables_session_affinity() -> Result<()> {- let client = Arc::new(PerRequestClient::default());+ let recorder = Arc::new(Recorder::default());let target = |name: &str| LlmTarget {semantic_name: name.to_string(),- llm_client: Some(client.clone()),};let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {judge_target: target("judge"),@@ -1228,25 +1247,31 @@ mod tests {},})?);- router- .clone()- .run(Context::default(), classify_session_request())- .await?;- router- .clone()- .run(Context::default(), classify_session_request())- .await?;+ let session_request = classify_session_request;+ drive(+ router.clone(),+ Context::default(),+ session_request(),+ recorder.serve(),+ )+ .await?;+ drive(+ router.clone(),+ Context::default(),+ session_request(),+ recorder.serve(),+ )+ .await?;- assert_eq!(client.calls(), vec!["judge", "efficient", "efficient"]);+ assert_eq!(recorder.calls(), vec!["judge", "efficient", "efficient"]);Ok(())}#[tokio::test]async fn classifier_config_reuses_message_hash_affinity_for_a_follow_up() -> Result<()> {- let client = Arc::new(PerRequestClient::default());+ let recorder = Arc::new(Recorder::default());let target = |name: &str| LlmTarget {semantic_name: name.to_string(),- llm_client: Some(client.clone()),};let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {judge_target: target("judge"),@@ -1260,16 +1285,22 @@ mod tests {},})?);- router- .clone()- .run(Context::default(), classify_request())- .await?;- router- .clone()- .run(Context::default(), classify_follow_up_request())- .await?;+ drive(+ router.clone(),+ Context::default(),+ classify_request(),+ recorder.serve(),+ )+ .await?;+ drive(+ router.clone(),+ Context::default(),+ classify_follow_up_request(),+ recorder.serve(),+ )+ .await?;- assert_eq!(client.calls(), vec!["judge", "efficient", "efficient"]);+ assert_eq!(recorder.calls(), vec!["judge", "efficient", "efficient"]);Ok(())}@@ -1313,7 +1344,6 @@ mod tests {fn invalid_classifier_config_is_rejected() -> Result<()> {let target = |name: &str| LlmTarget {semantic_name: name.to_string(),- llm_client: None,};for bad in [1.5, -0.1, f64::NAN, f64::INFINITY] {assert!(@@ -1765,57 +1795,54 @@ mod tests {use std::collections::VecDeque;- use switchyard_protocol::LlmClientError as ClientError;-use switchyard_protocol::Decision;- /// Serves each call with the next queued reply.- struct QueuedClient {- replies: Mutex<VecDeque<String>>,- }+ /// A queue of replies, drained in order.+ struct Queue(Mutex<VecDeque<String>>);- impl QueuedClient {+ impl Queue {fn new(replies: impl IntoIterator<Item = &'static str>) -> Arc<Self> {- Arc::new(Self {- replies: Mutex::new(replies.into_iter().map(String::from).collect()),- })+ Arc::new(Self(Mutex::new(+ replies.into_iter().map(String::from).collect(),+ )))}- }- #[async_trait]- impl RoutedLlmClient for QueuedClient {- async fn call(- &self,- _ctx: Context,- request: Request,- _decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, ClientError> {- let reply = self- .replies+ fn take(&self) -> String {+ self.0.lock().pop_front()- .unwrap_or_else(|| "unexpected call".to_string());- Ok(Response {- llm_response: LlmResponse::Agg(text_response(None, reply)),- metadata: request.metadata,- })+ .unwrap_or_else(|| "unexpected call".to_string())+ }+ }++ /// Serves the judge target from `judge` and every other target from `model`, each with+ /// its next queued reply.+ fn queued(model: Arc<Queue>, judge: Arc<Queue>) -> impl Serve {+ move |decision: Arc<dyn Decision>, request: Request| {+ let queue = if decision.selected_model() == "judge" {+ Arc::clone(&judge)+ } else {+ Arc::clone(&model)+ };+ async move {+ Ok(Response {+ llm_response: LlmResponse::Agg(text_response(None, queue.take())),+ metadata: request.metadata,+ })+ }}}/// Builds a router with escalation enabled (`confirmations=1` latches on the first verdict).- fn escalation_router(- client: Arc<dyn RoutedLlmClient>,- judge_client: Arc<dyn RoutedLlmClient>,- ) -> Result<Arc<LlmTaskClassifier>> {- let target = |name: &str, c: Arc<dyn RoutedLlmClient>| LlmTarget {+ fn escalation_router() -> Result<Arc<LlmTaskClassifier>> {+ let target = |name: &str| LlmTarget {semantic_name: name.to_string(),- llm_client: Some(c),};Ok(Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Escalation {- judge_target: target("judge", judge_client),- efficient_target: target("efficient", client.clone()),- capable_target: target("capable", client),+ judge_target: target("judge"),+ efficient_target: target("efficient"),+ capable_target: target("capable"),contract: ClassifierContractConfig::default(),config: EscalationJudgeConfig {confirmations: 1,@@ -1829,12 +1856,17 @@ mod tests {#[tokio::test]async fn escalation_router_serves_efficient_when_judge_declines() -> Result<()> {// Judge: no escalation. Expect the efficient response to be returned directly.- let judge_client = QueuedClient::new([r#"{"escalate":false,"reason":"progressing"}"#]);- let model_client = QueuedClient::new(["efficient answer"]);- let router = escalation_router(model_client, judge_client)?;- let request = classify_request();-- let (trace, response) = router.run(Context::default(), request).await?;+ let judge = Queue::new([r#"{"escalate":false,"reason":"progressing"}"#]);+ let model = Queue::new(["efficient answer"]);+ let router = escalation_router()?;++ let (trace, response) = drive(+ router,+ Context::default(),+ classify_request(),+ queued(model, judge),+ )+ .await?;// The efficient model is the serving target, and the response comes from its call.assert_eq!(trace.last().map(|d| d.selected_model()), Some("efficient"));@@ -1847,10 +1879,9 @@ mod tests {#[tokio::test]async fn escalation_config_overrides_the_packaged_prompt() -> Result<()> {- let client = Arc::new(PerRequestClient::default());+ let recorder = Arc::new(Recorder::default());let target = |name: &str| LlmTarget {semantic_name: name.to_string(),- llm_client: Some(client.clone()),};let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Escalation {judge_target: target("judge"),@@ -1864,9 +1895,15 @@ mod tests {max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,})?);- router.run(Context::default(), classify_request()).await?;+ drive(+ router,+ Context::default(),+ classify_request(),+ recorder.serve(),+ )+ .await?;- let prompts = client.judge_system_prompts();+ let prompts = recorder.judge_system_prompts();assert_eq!(prompts.len(), 1);assert_eq!(prompts[0], "Custom trajectory rubric.");Ok(())@@ -1875,13 +1912,18 @@ mod tests {#[tokio::test]async fn escalation_router_upgrades_to_capable_when_judge_escalates() -> Result<()> {// Judge: escalate. After the efficient call, the streak confirms and capable is served.- let judge_client = QueuedClient::new([r#"{"escalate":true,"reason":"stuck in a loop"}"#]);+ let judge = Queue::new([r#"{"escalate":true,"reason":"stuck in a loop"}"#]);// Efficient is called first (by the classifier), then capable is called by FallThrough.- let model_client = QueuedClient::new(["efficient draft", "capable answer"]);- let router = escalation_router(model_client, judge_client)?;- let request = classify_request();-- let (trace, response) = router.run(Context::default(), request).await?;+ let model = Queue::new(["efficient draft", "capable answer"]);+ let router = escalation_router()?;++ let (trace, response) = drive(+ router,+ Context::default(),+ classify_request(),+ queued(model, judge),+ )+ .await?;assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));assert_eq!(@@ -1895,19 +1937,25 @@ mod tests {async fn escalation_router_stays_capable_after_latch() -> Result<()> {// First turn: judge escalates and the streak latches.// Second turn: judge is not called again; capable is served directly.- let judge_client = QueuedClient::new([r#"{"escalate":true,"reason":"stuck"}"#]);- let model_client = QueuedClient::new(["efficient draft", "capable t1", "capable t2"]);- let router = escalation_router(model_client, judge_client)?;+ let judge = Queue::new([r#"{"escalate":true,"reason":"stuck"}"#]);+ let model = Queue::new(["efficient draft", "capable t1", "capable t2"]);+ let router = escalation_router()?;let session_request = classify_session_request();- router- .clone()- .run(Context::default(), session_request.clone())- .await?;- let (trace, _) = router- .clone()- .run(Context::default(), session_request)- .await?;+ drive(+ router.clone(),+ Context::default(),+ session_request.clone(),+ queued(Arc::clone(&model), Arc::clone(&judge)),+ )+ .await?;+ let (trace, _) = drive(+ router.clone(),+ Context::default(),+ session_request,+ queued(model, judge),+ )+ .await?;assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));Ok(())@@ -1918,39 +1966,22 @@ mod tests {// When the efficient model exceeds its context window inside score(), the classifier// must return capable rather than propagating the error — otherwise the client sees// HTTP 400 instead of a response from the strong model.- struct OverflowClient;- #[async_trait]- impl RoutedLlmClient for OverflowClient {- async fn call(- &self,- _ctx: Context,- _request: Request,- decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, LlmClientError> {- Err(LlmClientError::ContextWindowExceeded {+ let router = escalation_router()?;++ // Efficient overflows, capable answers, and the judge must never be called.+ let serve = |decision: Arc<dyn Decision>, _request: Request| async move {+ match decision.selected_model() {+ "efficient" => Err(LlmClientError::ContextWindowExceeded {model: decision.selected_model().to_string(),message: "prompt is too long".to_string(),- })+ }),+ "judge" => panic!("the judge must not be consulted when efficient overflows"),+ _ => Ok(reply("capable answer")),}- }-- let target = |name: &str, c: Arc<dyn RoutedLlmClient>| LlmTarget {- semantic_name: name.to_string(),- llm_client: Some(c),};- let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Escalation {- judge_target: target("judge", QueuedClient::new([])), // must not be called- efficient_target: target("efficient", Arc::new(OverflowClient)),- capable_target: target("capable", QueuedClient::new(["capable answer"])),- contract: ClassifierContractConfig::default(),- config: EscalationJudgeConfig {- confirmations: 1,- ..EscalationJudgeConfig::default()- },- max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,- })?);- let (trace, response) = router.run(Context::default(), classify_request()).await?;+ let (trace, response) =+ drive(router, Context::default(), classify_request(), serve).await?;assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));assert_eq!(diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rsindex 71118cd4f..66e2b9cdb 100644--- a/crates/libsy/src/algorithms/noop.rs+++ b/crates/libsy/src/algorithms/noop.rs@@ -79,6 +79,7 @@ mod tests {use switchyard_protocol::{LlmRequest, Message, Role};use super::*;+ use crate::core::testing::{drive, echo};#[tokio::test]async fn test_noop_algo() -> Result<()> {@@ -93,8 +94,10 @@ mod tests {metadata: None,};+ // `Noop` synthesizes its own response and never offloads a call, so `echo` is+ // never reached.let a: Arc<dyn Algorithm> = Arc::new(Noop {});- let (decisions, response) = a.run(Context::default(), request).await?;+ let (decisions, response) = drive(a, Context::default(), request, echo()).await?;let Some(decision) = decisions.first() else {panic!("Expected exactly one Decision");};diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rsindex 9643fc4b4..85817ba4d 100644--- a/crates/libsy/src/algorithms/passthrough.rs+++ b/crates/libsy/src/algorithms/passthrough.rs@@ -8,7 +8,7 @@ use std::sync::Arc;use switchyard_protocol::{Request, Response};use crate::Result;-use crate::core::algorithm::{Algorithm, Driver, LlmTarget};+use crate::core::algorithm::{Algorithm, Driver, LlmTarget, RoutedRequest};use switchyard_protocol::{Context, Decision};/// Routing algorithm that always calls one configured target.@@ -58,7 +58,11 @@ impl Algorithm for Passthrough {});driver.info(ctx.clone(), decision.clone()).await?;driver- .call_llm_target(ctx, &self.target, request, decision)+ .call_llm(RoutedRequest {+ request,+ decision,+ ctx,+ }).await}}@@ -69,29 +73,8 @@ mod tests {use super::Passthrough;use crate::core::algorithm::{Algorithm, LlmTarget};- use switchyard_protocol::{- Context, Decision, LlmResponse, Request, Response, RoutedLlmClient, completion_text,- text_request, text_response,- };-- /// Echoes the selected target so tests can inspect which target was called.- /// TODO: Duplicated from rand.rs- struct EchoClient;-- #[async_trait::async_trait]- impl RoutedLlmClient for EchoClient {- async fn call(- &self,- _ctx: Context,- _request: Request,- decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {- Ok(Response {- llm_response: LlmResponse::Agg(text_response(None, decision.selected_model())),- metadata: None,- })- }- }+ use crate::core::testing::{drive, echo};+ use switchyard_protocol::{Context, Request, completion_text, text_request};#[tokio::test]async fn test_passthrough() -> crate::Result<()> {@@ -103,9 +86,8 @@ mod tests {};let algorithm: Arc<dyn Algorithm> = Arc::new(Passthrough::new(LlmTarget {semantic_name: MODEL_ID.to_string(),- llm_client: Some(Arc::new(EchoClient)),}));- let (trace, response) = algorithm.run(Context::default(), request).await?;+ let (trace, response) = drive(algorithm, Context::default(), request, echo()).await?;assert_eq!(responsediff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rsindex 3f41500c5..caa3fecc5 100644--- a/crates/libsy/src/algorithms/rand.rs+++ b/crates/libsy/src/algorithms/rand.rs@@ -177,30 +177,13 @@ mod tests {use super::*;use std::collections::HashSet;- use switchyard_protocol::{Metadata, completion_text, text_request, text_response};+ use switchyard_protocol::{Metadata, completion_text, text_request};use crate::DriverError;use crate::algorithms::util::affinity::AffinityRouter;use crate::core::algorithm::LlmTarget;- use switchyard_protocol::{Decision, LlmResponse, Request, RoutedLlmClient, Signals};-- /// Echoes the selected target so tests can inspect which target was called.- struct EchoClient;-- #[async_trait]- impl RoutedLlmClient for EchoClient {- async fn call(- &self,- _ctx: Context,- _request: Request,- decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {- Ok(Response {- llm_response: LlmResponse::Agg(text_response(None, decision.selected_model())),- metadata: None,- })- }- }+ use crate::core::testing::{drive, echo};+ use switchyard_protocol::{Request, Signals};fn request() -> Request {Request {@@ -225,13 +208,11 @@ mod tests {.iter().map(|name| LlmTarget {semantic_name: (*name).to_string(),- llm_client: Some(Arc::new(EchoClient)),}).collect();LlmTargetSet::new(targets)}- /// Builds a random router whose targets all share an echo client.fn algorithm(names: &[&str], weights: Option<Vec<f64>>, seed: Option<u64>) -> Result<Random> {Random::new(target_set(names), weights, seed)}@@ -243,7 +224,8 @@ mod tests {async fn selected_models(algorithm: Arc<dyn Algorithm>, count: usize) -> Result<Vec<String>> {let mut selected = Vec::with_capacity(count);for _ in 0..count {- let (_, response) = algorithm.clone().run(Context::default(), request()).await?;+ let (_, response) =+ drive(algorithm.clone(), Context::default(), request(), echo()).await?;selected.push(response.llm_response@@ -258,7 +240,7 @@ mod tests {#[tokio::test]async fn single_target_is_always_selected_and_called() -> Result<()> {let algorithm = shared_algorithm(&["only/model"])?;- let (trace, response) = algorithm.run(Context::default(), request()).await?;+ let (trace, response) = drive(algorithm, Context::default(), request(), echo()).await?;assert_eq!(response@@ -279,7 +261,8 @@ mod tests {let algorithm = shared_algorithm(&names)?;for _ in 0..50 {- let (trace, response) = algorithm.clone().run(Context::default(), request()).await?;+ let (trace, response) =+ drive(algorithm.clone(), Context::default(), request(), echo()).await?;let selected = response.llm_response.as_agg()@@ -300,7 +283,8 @@ mod tests {let mut seen = HashSet::new();for _ in 0..100 {- let (_, response) = algorithm.clone().run(Context::default(), request()).await?;+ let (_, response) =+ drive(algorithm.clone(), Context::default(), request(), echo()).await?;seen.insert(response.llm_response@@ -364,10 +348,13 @@ mod tests {.with_classifier(random),);- let (_, first) = algorithm- .clone()- .run(Context::default(), request_for_session("session-1"))- .await?;+ let (_, first) = drive(+ algorithm.clone(),+ Context::default(),+ request_for_session("session-1"),+ echo(),+ )+ .await?;let selected = first.llm_response.as_agg()@@ -386,9 +373,13 @@ mod tests {Some(selected.to_string()));- let (_, second) = algorithm- .run(Context::default(), request_for_session("session-1"))- .await?;+ let (_, second) = drive(+ algorithm,+ Context::default(),+ request_for_session("session-1"),+ echo(),+ )+ .await?;assert_eq!(second.llm_response@@ -440,7 +431,7 @@ mod tests {#[tokio::test]async fn decision_is_inspectable_and_downcasts() -> Result<()> {let algorithm = shared_algorithm(&["only/model"])?;- let (trace, _) = algorithm.run(Context::default(), request()).await?;+ let (trace, _) = drive(algorithm, Context::default(), request(), echo()).await?;let decision = &trace[0];assert_eq!(decision.selected_model(), "only/model");diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rsindex c82577c5b..b7a370e5a 100644--- a/crates/libsy/src/algorithms/stage.rs+++ b/crates/libsy/src/algorithms/stage.rs@@ -223,22 +223,20 @@ mod tests {use parking_lot::Mutex;use serde_json::json;use switchyard_protocol::{- ContentBlock, LlmRequest, Message, Role, ToolCall, ToolResult, WireFormat, text_response,+ ContentBlock, LlmRequest, Message, Role, ToolCall, ToolResult, WireFormat,};use super::*;use crate::algorithms::util::stage::DECISION_SOURCE_KEY;- use crate::core::algorithm::{Algorithm, LlmTarget};+ use crate::core::algorithm::LlmTarget;use crate::core::classifier::Score;use crate::core::state::StateValue;- use switchyard_protocol::{- Context, Decision, LlmResponse, Metadata, Response, RoutedLlmClient,- };+ use crate::core::testing::{Serve, drive, reply};+ use switchyard_protocol::{Context, Decision, Metadata, Response};fn tier_target(name: &str) -> LlmTarget {LlmTarget {semantic_name: name.to_string(),- llm_client: None,}}@@ -329,7 +327,6 @@ mod tests {config.llm_fallback = Some(LlmFallback {judge_target: LlmTarget {semantic_name: "judge".to_string(),- llm_client: None,},config: TaskClassifierConfig {base_threshold: -0.1,@@ -360,16 +357,14 @@ mod tests {messages: Vec<String>,}- /// Records what each target receives. When called as the judge target it- /// replies with a structured verdict so the fallback classifier gets an answer- /// without a real model.+ /// Records what each target receives.#[derive(Default)]- struct RecordingClient {+ struct Recorder {calls: Mutex<Vec<Call>>,judge_p_solve: Mutex<f64>,}- impl RecordingClient {+ impl Recorder {fn routed(&self) -> Vec<Call> {self.calls.lock()@@ -378,55 +373,48 @@ mod tests {.cloned().collect()}- }- #[async_trait]- impl RoutedLlmClient for RecordingClient {- async fn call(- &self,- _ctx: Context,- request: Request,- decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {- let target = decision.selected_model().to_string();- self.calls.lock().push(Call {- target: target.clone(),- messages: request- .llm_request- .messages- .iter()- .filter_map(|message| message.text_content("|"))- .collect(),- });- let completion = if target == JUDGE {- let p_solve = *self.judge_p_solve.lock();- format!(- r#"{{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":{p_solve}}}"#- )- } else {- target- };- Ok(Response {- llm_response: LlmResponse::Agg(text_response(None, completion)),- metadata: None,- })+ /// Serves every call, recording it. The judge target gets a structured verdict+ /// back so the fallback classifier has an answer without a real model.+ fn serve(self: &Arc<Self>) -> impl Serve {+ let recorder = Arc::clone(self);+ move |decision: Arc<dyn Decision>, request: Request| {+ let recorder = Arc::clone(&recorder);+ async move {+ let target = decision.selected_model().to_string();+ recorder.calls.lock().push(Call {+ target: target.clone(),+ messages: request+ .llm_request+ .messages+ .iter()+ .filter_map(|message| message.text_content("|"))+ .collect(),+ });+ let completion = if target == JUDGE {+ let p_solve = *recorder.judge_p_solve.lock();+ format!(+ r#"{{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":{p_solve}}}"#+ )+ } else {+ target+ };+ Ok(reply(completion))+ }+ }}}- fn recording_target(client: &Arc<RecordingClient>, name: &str) -> LlmTarget {+ fn recording_target(name: &str) -> LlmTarget {LlmTarget {semantic_name: name.to_string(),- llm_client: Some(client.clone() as Arc<dyn RoutedLlmClient>),}}- fn recording_router(- client: Arc<RecordingClient>,- config: StageRouterConfig,- ) -> Result<Arc<StageRouter>> {+ fn recording_router(config: StageRouterConfig) -> Result<Arc<StageRouter>> {Ok(Arc::new(StageRouter::new(- recording_target(&client, "strong"),- recording_target(&client, "weak"),+ recording_target("strong"),+ recording_target("weak"),config,)?))}@@ -437,11 +425,11 @@ mod tests {c}- fn config_with_judge(client: &Arc<RecordingClient>, p_solve: f64) -> StageRouterConfig {- *client.judge_p_solve.lock() = p_solve;+ fn config_with_judge(recorder: &Arc<Recorder>, p_solve: f64) -> StageRouterConfig {+ *recorder.judge_p_solve.lock() = p_solve;let mut c = config();c.llm_fallback = Some(LlmFallback {- judge_target: recording_target(client, JUDGE),+ judge_target: recording_target(JUDGE),config: TaskClassifierConfig {base_threshold: 0.5,recent_turn_window: Some(3),@@ -502,14 +490,20 @@ mod tests {#[tokio::test]async fn a_signal_driven_escalation_hands_the_note_to_the_model() -> Result<()> {- let client = Arc::new(RecordingClient::default());- let router = recording_router(client.clone(), config_with_notes())?;+ let recorder = Arc::new(Recorder::default());+ let router = recording_router(config_with_notes())?;let ctx = Context::default();- router.clone().run(ctx.clone(), turn_request(false)).await?;- router.run(ctx, turn_request(true)).await?;+ drive(+ router.clone(),+ ctx.clone(),+ turn_request(false),+ recorder.serve(),+ )+ .await?;+ drive(router.clone(), ctx, turn_request(true), recorder.serve()).await?;- let calls = client.routed();+ let calls = recorder.routed();assert_eq!(calls[0].target, "weak");assert_eq!(calls[1].target, "strong");assert!(@@ -530,49 +524,67 @@ mod tests {#[tokio::test]async fn the_judge_decides_a_turn_the_signals_leave_undecided() -> Result<()> {- let client = Arc::new(RecordingClient::default());- let router = recording_router(client.clone(), config_with_judge(&client, 0.1))?;+ let recorder = Arc::new(Recorder::default());+ let router = recording_router(config_with_judge(&recorder, 0.1))?;- router.run(Context::default(), turn_request(false)).await?;+ drive(+ router.clone(),+ Context::default(),+ turn_request(false),+ recorder.serve(),+ )+ .await?;assert!(- client.calls.lock().iter().any(|c| c.target == JUDGE),+ recorder.calls.lock().iter().any(|c| c.target == JUDGE),"the judge should be consulted on an undecided turn");- assert_eq!(client.routed()[0].target, "strong");+ assert_eq!(recorder.routed()[0].target, "strong");Ok(())}#[tokio::test]async fn a_decisive_signal_never_reaches_the_judge() -> Result<()> {- let client = Arc::new(RecordingClient::default());- let router = recording_router(client.clone(), config_with_judge(&client, 0.9))?;+ let recorder = Arc::new(Recorder::default());+ let router = recording_router(config_with_judge(&recorder, 0.9))?;- router.run(Context::default(), turn_request(true)).await?;+ drive(+ router.clone(),+ Context::default(),+ turn_request(true),+ recorder.serve(),+ )+ .await?;assert!(- !client.calls.lock().iter().any(|c| c.target == JUDGE),+ !recorder.calls.lock().iter().any(|c| c.target == JUDGE),"a resolved turn should not pay for a judge call");- assert_eq!(client.routed()[0].target, "strong");+ assert_eq!(recorder.routed()[0].target, "strong");Ok(())}#[tokio::test]async fn the_judges_verdict_is_not_pinned_to_the_session() -> Result<()> {- let client = Arc::new(RecordingClient::default());- let router = recording_router(client.clone(), config_with_judge(&client, 0.1))?;+ let recorder = Arc::new(Recorder::default());+ let router = recording_router(config_with_judge(&recorder, 0.1))?;let ctx = Context::default();- router.clone().run(ctx.clone(), turn_request(false)).await?;- *client.judge_p_solve.lock() = 0.9;- router.run(ctx, turn_request(false)).await?;-- let routed = client.routed();+ drive(+ router.clone(),+ ctx.clone(),+ turn_request(false),+ recorder.serve(),+ )+ .await?;+ *recorder.judge_p_solve.lock() = 0.9;+ drive(router.clone(), ctx, turn_request(false), recorder.serve()).await?;++ let routed = recorder.routed();assert_eq!(routed[0].target, "strong");assert_eq!(routed[1].target, "weak");assert_eq!(- client+ recorder.calls.lock().iter()@@ -586,23 +598,35 @@ mod tests {#[tokio::test]async fn a_judge_that_cannot_tell_lands_on_the_picker_default() -> Result<()> {- let client = Arc::new(RecordingClient::default());- let router = recording_router(client.clone(), config_with_judge(&client, 42.0))?;-- router.run(Context::default(), turn_request(false)).await?;-- assert_eq!(client.routed()[0].target, "weak");+ let recorder = Arc::new(Recorder::default());+ let router = recording_router(config_with_judge(&recorder, 42.0))?;++ drive(+ router.clone(),+ Context::default(),+ turn_request(false),+ recorder.serve(),+ )+ .await?;++ assert_eq!(recorder.routed()[0].target, "weak");Ok(())}#[tokio::test]async fn the_judge_reads_the_window_it_was_configured_with() -> Result<()> {- let client = Arc::new(RecordingClient::default());- let router = recording_router(client.clone(), config_with_judge(&client, 0.9))?;-- router.run(Context::default(), turn_request(false)).await?;-- let judged = client+ let recorder = Arc::new(Recorder::default());+ let router = recording_router(config_with_judge(&recorder, 0.9))?;++ drive(+ router.clone(),+ Context::default(),+ turn_request(false),+ recorder.serve(),+ )+ .await?;++ let judged = recorder.calls.lock().iter()diff --git a/crates/libsy/src/algorithms/subagent_affinity_tests.rs b/crates/libsy/src/algorithms/subagent_affinity_tests.rsindex 1e7198887..87ef0fcdc 100644--- a/crates/libsy/src/algorithms/subagent_affinity_tests.rs+++ b/crates/libsy/src/algorithms/subagent_affinity_tests.rs@@ -15,31 +15,13 @@ use super::fall_through::FallThrough;use super::util::affinity::AffinityRouter;use super::util::subagent::SubagentOverride;use crate::Result;-use crate::core::algorithm::{Algorithm, Driver, LlmTarget, LlmTargetSet};+use crate::core::algorithm::{Driver, LlmTarget, LlmTargetSet};use crate::core::classifier::{Classification, Classifier, Score};+use crate::core::testing::{drive, echo};use switchyard_protocol::{- Context, Decision, LlmResponse, Metadata, Request, Response, RoutedLlmClient, completion_text,- slice_to_header_map, text_request, text_response,+ Context, Metadata, Request, Response, completion_text, slice_to_header_map, text_request,};-/// A client that echoes the routed target name back as the completion.-struct EchoClient;--#[async_trait]-impl RoutedLlmClient for EchoClient {- async fn call(- &self,- _ctx: Context,- _request: Request,- decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {- Ok(Response {- llm_response: LlmResponse::Agg(text_response(None, decision.selected_model())),- metadata: None,- })- }-}-/// The cascade's terminal classifier: always picks the orchestrator.struct AlwaysOrchestrator;@@ -67,7 +49,6 @@ fn targets() -> LlmTargetSet {.iter().map(|name| LlmTarget {semantic_name: (*name).to_string(),- llm_client: Some(Arc::new(EchoClient) as Arc<dyn RoutedLlmClient>),}).collect(),)@@ -96,10 +77,7 @@ fn router() -> Arc<FallThrough> {/// Runs one turn, returning the target that served it.async fn turn(router: &Arc<FallThrough>, headers: &[(&str, &str)]) -> Result<String> {- let (_, response) = router- .clone()- .run(Context::default(), request(headers))- .await?;+ let (_, response) = drive(router.clone(), Context::default(), request(headers), echo()).await?;Ok(response.llm_response.as_agg()diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rsindex 3898c44fb..2f281187e 100644--- a/crates/libsy/src/algorithms/util/llm_judge.rs+++ b/crates/libsy/src/algorithms/util/llm_judge.rs@@ -18,7 +18,7 @@ use switchyard_protocol::{};use super::classifier_contract::ClassifierContract;-use crate::core::algorithm::{Driver, LlmTarget};+use crate::core::algorithm::{Driver, LlmTarget, RoutedRequest};use crate::core::classifier::{Classification, Classifier};use crate::core::state::State;use crate::{LibsyError, Result};@@ -227,14 +227,13 @@ wherelet judge_model = self.target.semantic_name.as_str();let response = driver- .call_llm_target(- Context::default(),- &self.target,- self.judge.build_request(state, request),- Arc::new(JudgeDecision {+ .call_llm(RoutedRequest {+ request: self.judge.build_request(state, request),+ decision: Arc::new(JudgeDecision {model: self.target.semantic_name.to_string(),}),- )+ ctx: Context::default(),+ }).await.inspect_err(|error| report_fail_open(judge_model, error, libsy_error_reason(error))).ok()?;@@ -409,7 +408,6 @@ mod tests {TestJudge,LlmTarget {semantic_name: "judge".to_string(),- llm_client: None,},TestPolicy,)diff --git a/crates/libsy/src/core.rs b/crates/libsy/src/core.rsindex b5655269e..e922faf13 100644--- a/crates/libsy/src/core.rs+++ b/crates/libsy/src/core.rs@@ -6,6 +6,8 @@//! pump in [`driver`]. Algorithm implementations live in [`crate::algorithms`].mod driver;+#[cfg(test)]+pub(crate) mod testing;pub mod algorithm;pub mod classifier;diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex 0d4e76f9f..d7179eb24 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -7,6 +7,7 @@use std::{collections::{HashMap, HashSet},+ future::Future,pin::Pin,sync::Arc,time::{Duration, Instant},@@ -25,8 +26,7 @@ use tracing::Instrument;/// [`switchyard_protocol::LlmResponse`] carries either a live/// [`switchyard_protocol::LlmResponseStream`] or the terminal aggregate.use switchyard_protocol::{- Context, Decision, LlmClientError, Request, Response, RoutedLlmClient, RoutingFallbackReason,- Signals, Usage,+ Context, Decision, LlmClientError, Request, Response, RoutingFallbackReason, Signals, Usage,};use super::driver::{DriverRequest, DriverStep, TypeErasedDriver};@@ -84,13 +84,8 @@ pub struct RoutedRequest {pub request: Request,/// The routing decision behind this call; `selected_model()` is the model to hit.pub decision: Arc<dyn Decision>,- /// The client that serves this call by default, or `None` when the routed target- /// had no client. Rides along on the offloaded call so a host driving the stream- /// can serve it by default or override it with its own transport.- pub default_client: Option<Arc<dyn RoutedLlmClient>>,- /// The request's cross-cutting context, carried through the offload so whoever- /// serves the call (libsy's own `run`, or a host driving the stream) hands it to- /// [`RoutedLlmClient::call`].+ /// The request's cross-cutting context, carried through the offload so the host+ /// serving the call can pass it to its client.pub ctx: Context,}@@ -100,7 +95,8 @@ pub struct RoutedRequest {/// routed request ([`get_routed`](Self::get_routed)) and the decision behind it/// ([`get_decision`](Self::get_decision)), performs (or delegates) the model call, and/// fulfills it with [`respond`](Self::respond) — unblocking the algorithm's-/// [`Driver::call_llm`] on the other side.+/// [`Driver::call_llm`] on the other side. `switchyard-llm-client`'s `run` is the+/// ready-made consumer that does this for you.pub struct CallLlmRequest {inner: DriverRequest,routed: RoutedRequest,@@ -119,9 +115,8 @@ impl CallLlmRequest {Self { inner, routed }}- /// The routed request the host should serve. Its- /// [`default_client`](RoutedRequest::default_client) serves the call by default,- /// and its `decision.selected_model()` names the model to hit.+ /// The routed request the host should serve; its `decision.selected_model()` names+ /// the model to hit.pub fn get_routed(&self) -> &RoutedRequest {&self.routed}@@ -146,10 +141,10 @@ impl CallLlmRequest {/// The offload channel handed to an algorithm's/// [`create_run_task`](Algorithm::create_run_task). The algorithm makes model calls-/// with [`call_llm_target`](Self::call_llm_target) (or [`call_llm`](Self::call_llm)) and-/// publishes its [`Decision`]s with [`info`](Self::info); each call is offloaded to the-/// request's [`Step`] stream and awaits the consumer's response. The step channel is-/// bounded, so the consumer paces the algorithm one step at a time.+/// with [`call_llm`](Self::call_llm) and publishes its [`Decision`]s with+/// [`info`](Self::info); each call is offloaded to the request's [`Step`] stream and+/// awaits the consumer's response. The step channel is bounded, so the consumer paces+/// the algorithm one step at a time.#[derive(Clone)]pub struct Driver {driver: TypeErasedDriver,@@ -192,8 +187,8 @@ impl Driver {/// The await is wrapped in a `libsy.llm_call` span measuring *fulfillment* as/// the algorithm observes it (host queueing/serving included; a streamed/// response resolves when its stream handle arrives); latency, outcome, and- /// token usage are recorded when it resolves. The provider call itself gets a- /// `libsy.client_call` span when [`Algorithm::run`] serves it.+ /// token usage are recorded when it resolves. The provider call itself is the+ /// host's, and is instrumented by whoever makes it.#[tracing::instrument(target = "libsy",name = "libsy.llm_call",@@ -252,27 +247,6 @@ impl Driver {result}- /// Offload a call to `target`: pair `request` with `decision` and the target's- /// default client into a [`RoutedRequest`], then publish it (see- /// [`call_llm`](Self::call_llm)). The convenience most algorithms use;- /// `decision.selected_model()` names the model to hit, and `request`'s- /// `model` is left untouched.- pub async fn call_llm_target(- &self,- ctx: Context,- target: &LlmTarget,- request: Request,- decision: Arc<dyn Decision>,- ) -> Result<Response> {- self.call_llm(RoutedRequest {- request,- decision,- default_client: target.llm_client.clone(),- ctx,- })- .await- }-/// Publish a routing [`Decision`] as a [`Step::Decision`] on the stream./// Each successfully published decision is counted and logged with its/// reasoning; a decision the stream never accepted is not recorded.@@ -328,9 +302,8 @@ impl Default for Driver {/// One item in the stream returned by `Driver::stream` / [`Algorithm::run_stream`].pub enum Step {- /// The algorithm needs this model call performed. The host serves it (optionally- /// via [`RoutedRequest::default_client`]) and fulfills it with- /// [`CallLlmRequest::respond`]. Boxed: it is by far the largest variant.+ /// The algorithm needs this model call performed. The host serves it and fulfills+ /// it with [`CallLlmRequest::respond`]. Boxed: it is by far the largest variant.CallLlm(Box<CallLlmRequest>),/// A routing decision the algorithm made, published via [`Driver::info`] as it/// happens (rather than collected into a trace returned at the end).@@ -339,6 +312,64 @@ pub enum Step {ReturnToAgent(Box<Response>),}+/// Drive [`Algorithm::run_stream`] to completion, handing each offloaded call to `serve`.+///+/// Returns the final [`Response`] and the trace of [`Decision`]s the algorithm published.+/// `observer`, when present, is passed through to `run_stream`.+///+/// `serve` owns the call: it performs it however the host likes and must fulfill the promise+/// with [`CallLlmRequest::respond`]. A failed *model* call belongs in `respond` — the+/// algorithm may route around it. Returning `Err` from `serve` aborts the whole run, so+/// reserve it for infrastructure failures. Calls are served concurrently, so an algorithm+/// that offloads several at once (hedging, fan-out) gets real parallelism.+///+/// libsy performs no I/O; this is only the mechanics of consuming its own step stream, kept+/// here so every host does not reimplement the same loop. `switchyard-llm-client`'s `run`+/// is this function plus an HTTP client.+pub async fn drive<F, Fut>(+ algorithm: Arc<dyn Algorithm>,+ ctx: Context,+ request: Request,+ observer: Option<RunObserver>,+ serve: F,+) -> Result<(Vec<Arc<dyn Decision>>, Response)>+where+ F: Fn(CallLlmRequest) -> Fut,+ Fut: Future<Output = Result<()>>,+{+ let stream = algorithm.run_stream(ctx, request, observer);+ tokio::pin!(stream);++ let mut trace: Vec<Arc<dyn Decision>> = Vec::new();+ let mut in_flight = futures::stream::FuturesUnordered::new();+ let mut final_response: Option<Response> = None;++ loop {+ tokio::select! {+ Some(result) = in_flight.next() => match result {+ Ok(()) => {}, // CallLlm completed successfully+ Err(err) => return Err(err), // CallLlm failed, propagate the error+ },+ step = stream.next() => {+ match step {+ None => break, // stream has ended, no more steps+ Some(item) => match item? {+ Step::CallLlm(call) => in_flight.push(serve(*call)),+ Step::Decision(decision) => trace.push(decision),+ Step::ReturnToAgent(response) => {+ final_response = Some(*response);+ break;+ }+ }+ }+ },+ }+ }+ final_response+ .map(|response| (trace, response))+ .ok_or(LibsyError::MissingFinalResponse)+}+/// Abort guardstruct AbortOnDrop(tokio::task::AbortHandle);@@ -348,19 +379,15 @@ impl Drop for AbortOnDrop {}}-/// A named routing target: a `semantic_name` an algorithm routes by, and an optional-/// [`RoutedLlmClient`] to serve its calls. An algorithm hands a target to-/// [`Driver::call_llm_target`]; the client rides along as-/// [`RoutedRequest::default_client`] for the stream consumer to serve or override.+/// A named routing target an algorithm routes by. Serving its calls is the stream+/// consumer's concern: the target's name reaches the consumer as+/// `decision.selected_model()` on the offloaded [`RoutedRequest`].#[derive(Clone)]pub struct LlmTarget {/// The routing label an algorithm selects this target by — a logical tier like/// `"strong"`, or the model id when they coincide. Mapping it to a provider model- /// id is the client's concern, never the algorithm's.+ /// id is the consumer's concern, never the algorithm's.pub semantic_name: String,- /// The client that serves this target's calls by default, or `None` (then the- /// stream consumer must serve them).- pub llm_client: Option<Arc<dyn RoutedLlmClient>>,}/// The set of targets an algorithm may route among. An algorithm is constructed@@ -567,7 +594,11 @@ pub(crate) async fn call_llm_with_fallback() -> Result<Response> {loop {let result = driver- .call_llm_target(ctx.clone(), &target, request.clone(), decision.clone())+ .call_llm(RoutedRequest {+ request: request.clone(),+ decision: decision.clone(),+ ctx: ctx.clone(),+ }).await;let Err(error) = result else { return result };let Some((failed, reason)) = classify_fallback(&error) else {@@ -592,8 +623,9 @@ pub(crate) async fn call_llm_with_fallback(}/// An optimization strategy. Implement [`create_run_task`](Self::create_run_task);-/// callers drive it with the provided [`run`](Self::run) (serve calls, get the answer)-/// or [`run_stream`](Self::run_stream) (drive the [`Step`] stream yourself).+/// callers drive it with [`run_stream`](Self::run_stream), serving each [`Step::CallLlm`]+/// it emits. `switchyard-llm-client`'s `run` is the ready-made consumer that does this+/// over HTTP.////// Methods take `self: Arc<Self>`: one algorithm (`Arc<dyn Algorithm>`) is shared across/// requests and run concurrently, so it owns its thread-safety and any shared state.@@ -608,9 +640,9 @@ pub(crate) async fn call_llm_with_fallback(/// # Observability////// [`run_stream`](Self::run_stream) creates a `libsy.run` span, and each offloaded model-/// call creates a `libsy.llm_call` span. [`run`](Self::run) additionally wraps calls it-/// serves through a target's default client in `libsy.client_call`. Decisions and failures-/// are emitted through `tracing`; metrics use the global OpenTelemetry meter provider.+/// call creates a `libsy.llm_call` span. Decisions and failures are emitted through+/// `tracing`; metrics use the global OpenTelemetry meter provider. The provider call+/// itself belongs to the host, and is instrumented by whoever makes it.#[async_trait]pub trait Algorithm: Send + Sync + 'static {/// Stable, low-cardinality name identifying this algorithm — the@@ -618,11 +650,11 @@ pub trait Algorithm: Send + Sync + 'static {/// emits for its runs.fn name(&self) -> &str;- /// Run one request to completion: make model calls with [`Driver::call_llm_target`],+ /// Run one request to completion: make model calls with [`Driver::call_llm`],/// publish [`Decision`]s with [`Driver::info`], and return the final [`Response`].- /// The method an algorithm implements; [`run`](Self::run) / [`run_stream`](Self::run_stream)- /// drive it. `ctx` carries the request's cross-cutting values (today: the- /// algorithm's telemetry label in [`Context::values`]).+ /// The method an algorithm implements; [`run_stream`](Self::run_stream) drives it.+ /// `ctx` carries the request's cross-cutting values (today: the algorithm's+ /// telemetry label in [`Context::values`]).async fn create_run_task(self: Arc<Self>,ctx: Context,@@ -703,136 +735,12 @@ pub trait Algorithm: Send + Sync + 'static {step}))}-- /// Process a request to completion, returning the final [`Response`] and the trace of- /// [`Decision`]s the algorithm made along the way.- ///- async fn run(- self: Arc<Self>,- ctx: Context,- request: Request,- ) -> Result<(Vec<Arc<dyn Decision>>, Response)> {- self.run_observed(ctx, request, None).await- }-- /// Process a request to completion while reporting each model call to `observer`.- async fn run_observed(- self: Arc<Self>,- ctx: Context,- request: Request,- observer: Option<RunObserver>,- ) -> Result<(Vec<Arc<dyn Decision>>, Response)> {- // Serve one offloaded call with its target's default client. A failed *model*- // call is forwarded to the algorithm via `respond`; this errors only on an- // infrastructure failure (no default client, or the promise was dropped).- // `serve` makes the one API call libsy itself performs, so it gets its- // own `libsy.client_call` span.- #[tracing::instrument(- target = "libsy",- name = "libsy.client_call",- skip_all,- fields(- algorithm = observability::algorithm_label(&call.get_routed().ctx),- switchyard.algorithm = observability::algorithm_label(&call.get_routed().ctx),- switchyard.routing.tier = tracing::field::Empty,- selected_model = call.get_decision().selected_model(),- otel.kind = "client",- otel.name = %format_args!("chat {}", call.get_decision().selected_model()),- openinference.span.kind = "LLM",- gen_ai.operation.name = "chat",- gen_ai.request.model = call.get_decision().selected_model(),- gen_ai.request.stream = tracing::field::Empty,- gen_ai.request.temperature = tracing::field::Empty,- gen_ai.request.top_p = tracing::field::Empty,- gen_ai.request.top_k = tracing::field::Empty,- gen_ai.request.max_tokens = tracing::field::Empty,- gen_ai.request.reasoning.level = tracing::field::Empty,- gen_ai.output.type = tracing::field::Empty,- gen_ai.conversation.id = tracing::field::Empty,- server.address = tracing::field::Empty,- server.port = tracing::field::Empty,- gen_ai.response.id = tracing::field::Empty,- gen_ai.response.model = tracing::field::Empty,- gen_ai.usage.input_tokens = tracing::field::Empty,- gen_ai.usage.output_tokens = tracing::field::Empty,- gen_ai.usage.cache_read.input_tokens = tracing::field::Empty,- gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty,- gen_ai.usage.reasoning.output_tokens = tracing::field::Empty,- outcome = tracing::field::Empty,- otel.status_code = tracing::field::Empty,- error.type = tracing::field::Empty,- error = tracing::field::Empty,- )- )]- async fn serve(call: CallLlmRequest) -> Result<()> {- let span = tracing::Span::current();- observability::record_gen_ai_request(&span, &call.get_routed().request.llm_request);- if let Some(tier) = call.get_decision().routing_tier() {- span.record("switchyard.routing.tier", tier);- }- if let Some(session_id) = call- .get_routed()- .request- .metadata- .as_ref()- .and_then(|metadata| metadata.session_id.as_deref())- {- span.record("gen_ai.conversation.id", session_id);- }- let routed = call.get_routed().clone();- let target = routed.decision.selected_model().to_string();- let client =- routed- .default_client- .clone()- .ok_or_else(|| LibsyError::MissingClient {- target: target.clone(),- })?;- let result = client- .call(routed.ctx, routed.request, routed.decision)- .await- .map_err(|source| LibsyError::client_call(target, source));- let result = observability::observe_client_call(result);- call.respond(result)- }-- let stream = self.run_stream(ctx, request, observer);- tokio::pin!(stream);-- let mut trace: Vec<Arc<dyn Decision>> = Vec::new();- let mut in_flight = futures::stream::FuturesUnordered::new();- let mut final_response: Option<Response> = None;-- loop {- tokio::select! {- Some(result) = in_flight.next() => match result {- Ok(()) => {}, // CallLlm completed successfully- Err(err) => return Err(err), // CallLlm failed, propagate the error- },- step = stream.next() => {- match step {- None => break, // stream has ended, no more steps- Some(item) => match item? {- Step::CallLlm(call) => in_flight.push(serve(*call)),- Step::Decision(decision) => trace.push(decision),- Step::ReturnToAgent(response) => {- final_response = Some(*response);- break;- }- }- }- },- }- }- final_response- .map(|response| (trace, response))- .ok_or(LibsyError::MissingFinalResponse)- }}#[cfg(test)]mod tests {use super::*;+ use crate::core::testing::{Serve, ServeResult, drive, drive_observed, echo, reply};use futures::StreamExt;use switchyard_protocol::{LlmResponse, LlmResponseChunk, completion_text, text_request, text_response,@@ -901,28 +809,6 @@ mod tests {);}- /// Mock client that echoes back the target name it was called with.- struct EchoClient;-- #[async_trait]- impl RoutedLlmClient for EchoClient {- async fn call(- &self,- _ctx: Context,- _request: Request,- decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, LlmClientError> {- // Echo back the model the algorithm routed to (the decision's selection).- Ok(Response {- llm_response: LlmResponse::Agg(text_response(- None,- decision.selected_model().to_string(),- )),- metadata: None,- })- }- }-/// Trivial decision + algo used only to exercise the orchestrator: calls the/// first target and returns its response with a one-item trace.struct TestDecision {@@ -968,7 +854,11 @@ mod tests {});driver.info(ctx.clone(), decision.clone()).await?;driver- .call_llm_target(ctx, &target, request, decision)+ .call_llm(RoutedRequest {+ request,+ decision,+ ctx,+ }).await}}@@ -986,13 +876,11 @@ mod tests {}}- /// `(name, has_client)` — `has_client: false` builds a target with no default client.- fn target_set(names: &[(&str, bool)]) -> LlmTargetSet {+ fn target_set(names: &[&str]) -> LlmTargetSet {let targets = names.iter()- .map(|(name, has_client)| LlmTarget {+ .map(|name| LlmTarget {semantic_name: name.to_string(),- llm_client: has_client.then(|| Arc::new(EchoClient) as Arc<dyn RoutedLlmClient>),}).collect();LlmTargetSet::new(targets)@@ -1003,9 +891,14 @@ mod tests {let observations = Arc::new(Mutex::new(Vec::new()));let observed = observations.clone();let observer: RunObserver = Arc::new(move |observation| observed.lock().push(observation));- let (_, response) = orch(target_set(&[("direct/model", true)]))- .run_observed(Context::default(), request(), Some(observer))- .await?;+ let (_, response) = drive_observed(+ orch(target_set(&["direct/model"])),+ Context::default(),+ request(),+ Some(observer),+ echo(),+ )+ .await?;assert_eq!(response.llm_response.as_agg().map(completion_text),Some("direct/model".to_string())@@ -1035,48 +928,29 @@ mod tests {));}- /// Client that serves a call as a token stream — its `call` returns- /// [`LlmResponse::Stream`] replaying `chunks` in order (as `Ok` items).- struct StreamingClient {- chunks: Vec<LlmResponseChunk>,- }-- #[async_trait]- impl RoutedLlmClient for StreamingClient {- async fn call(- &self,- _ctx: Context,- _request: Request,- _decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, LlmClientError> {- let stream = futures::stream::iter(- self.chunks- .clone()- .into_iter()- .map(|chunk| Ok(chunk.into())),- )- .boxed();- Ok(Response {- llm_response: LlmResponse::Stream(stream),- metadata: None,- })- }- }-- /// Build a single-target algo whose one target streams `chunks`.- fn streaming_orch(chunks: Vec<LlmResponseChunk>) -> Arc<dyn Algorithm> {- let target = LlmTarget {- semantic_name: "stream/model".to_string(),- llm_client: Some(Arc::new(StreamingClient { chunks }) as Arc<dyn RoutedLlmClient>),+ /// Build a single-target algo, plus a `serve` that answers it as a token stream+ /// replaying `chunks` in order (as `Ok` items).+ fn streaming_orch(chunks: Vec<LlmResponseChunk>) -> (Arc<dyn Algorithm>, impl Serve) {+ let algo = orch(target_set(&["stream/model"]));+ let serve = move |_decision: Arc<dyn Decision>, _request: Request| {+ let chunks = chunks.clone();+ async move {+ let stream =+ futures::stream::iter(chunks.into_iter().map(|chunk| Ok(chunk.into()))).boxed();+ Ok(Response {+ llm_response: LlmResponse::Stream(stream),+ metadata: None,+ })+ }};- orch(LlmTargetSet::new(vec![target]))+ (algo, serve)}#[tokio::test]async fn run_returns_a_streamed_response_the_caller_aggregates() -> Result<()> {// A streaming client -> its chunks flow through the promise and `ReturnToAgent`,// and `run` returns the live stream untouched for the caller to fold.- let orch = streaming_orch(vec![+ let (orch, serve) = streaming_orch(vec![LlmResponseChunk::MessageStart {id: Some("m1".to_string()),model: Some("stream/model".to_string()),@@ -1093,8 +967,8 @@ mod tests {reason: Some("stop".to_string()),},]);- let (trace, response) = orch.run(Context::default(), request()).await?;- // `run` handed back the live stream; the caller folds it to a buffered aggregate.+ let (trace, response) = drive(orch, Context::default(), request(), serve).await?;+ // The run handed back the live stream; the caller folds it to a buffered aggregate.let agg = response.llm_response.into_agg()@@ -1108,9 +982,9 @@ mod tests {#[tokio::test]async fn aggregating_a_streamed_response_propagates_a_mid_stream_error() -> Result<()> {- // `run` succeeds and returns the stream; the in-band `Error` chunk surfaces only+ // The run succeeds and returns the stream; the in-band `Error` chunk surfaces only// when the caller aggregates it.- let orch = streaming_orch(vec![+ let (orch, serve) = streaming_orch(vec![LlmResponseChunk::TextDelta {index: 0,text: "partial".to_string(),@@ -1119,7 +993,7 @@ mod tests {message: "upstream exploded".to_string(),},]);- let (_, response) = orch.run(Context::default(), request()).await?;+ let (_, response) = drive(orch, Context::default(), request(), serve).await?;match response.llm_response.into_agg().await {Ok(_) => panic!("expected a mid-stream error, got an aggregate"),Err(err) => {@@ -1131,13 +1005,10 @@ mod tests {#[tokio::test]async fn run_offloads_via_promise_then_returns_to_agent() -> Result<()> {- // A client-less target -> its call is offloaded via a promise the- // orchestrator surfaces as a `CallLlm` step for us to fulfill.- let stream = orch(target_set(&[("offload/model", false)])).run_stream(- Context::default(),- request(),- None,- );+ // Every call is offloaded via a promise the orchestrator surfaces as a+ // `CallLlm` step for us to fulfill.+ let stream =+ orch(target_set(&["offload/model"])).run_stream(Context::default(), request(), None);tokio::pin!(stream);let mut saw_call = false;@@ -1181,61 +1052,15 @@ mod tests {}#[tokio::test]- async fn client_backed_target_offloads_with_a_default_client() -> Result<()> {- // Every call now offloads to the stream; a client-backed target rides its- // client along as `default_client` so the consumer can serve it by default.- let stream = orch(target_set(&[("direct/model", true)])).run_stream(+ async fn a_driven_run_returns_the_trace_and_the_final_response() -> Result<()> {+ let (trace, response) = drive(+ orch(target_set(&["direct/model"])),Context::default(),request(),- None,- );- tokio::pin!(stream);-- let mut final_completion = None;- while let Some(step) = stream.next().await {- match step? {- Step::CallLlm(call) => {- let routed = call.get_routed().clone();- let client = routed- .default_client- .clone()- .ok_or_else(|| test_error("expected a default client"))?;- let target = routed.decision.selected_model().to_string();- let result = client- .call(routed.ctx, routed.request, routed.decision)- .await- .map_err(|error| LibsyError::client_call(target, error));- call.respond(result)?;- }- Step::Decision(_) => {}- Step::ReturnToAgent(response) => {- final_completion = Some(- response- .llm_response- .as_agg()- .map(completion_text)- .unwrap_or_default(),- );- }- }- }-- // EchoClient echoes the model name back as the completion.- assert_eq!(- final_completion.ok_or_else(|| test_error("no ReturnToAgent"))?,- "direct/model"- );- Ok(())- }-- #[tokio::test]- async fn run_returns_the_response_when_all_targets_have_clients() -> Result<()> {- // Every target has a client, so run serves every call via the- // default client and returns the trace + final response.- let (trace, response) = orch(target_set(&[("direct/model", true)]))- .run(Context::default(), request())- .await?;- // TestAlgo calls the first target; EchoClient echoes its name.+ echo(),+ )+ .await?;+ // TestAlgo calls the first target; `echo` answers with its name.assert_eq!(response.llm_response@@ -1248,22 +1073,6 @@ mod tests {Ok(())}- #[tokio::test]- async fn run_errors_when_a_target_lacks_a_client() -> Result<()> {- // A client-less target has no default client to serve its offloaded call, so- // driving it to completion errors.- let error = orch(target_set(&[("offload/model", false)]))- .run(Context::default(), request())- .await- .err()- .ok_or_else(|| test_error("expected a missing-client error"))?;- assert!(matches!(- error,- LibsyError::MissingClient { target } if target == "offload/model"- ));- Ok(())- }-#[tokio::test(flavor = "multi_thread", worker_threads = 12)]async fn requests_are_processed_in_parallel() -> Result<()> {use std::time::Duration;@@ -1271,49 +1080,27 @@ mod tests {const N: usize = 12;- // A client that blocks until all N concurrent calls have arrived. If- // requests were serialized (one algorithm behind a `Mutex`), only one- // call could be in flight, the barrier would never reach N, and the test- // would time out. It passes only because the shared algorithm is driven- // concurrently across requests.- struct BarrierClient {- barrier: Arc<Barrier>,- }-- #[async_trait]- impl RoutedLlmClient for BarrierClient {- async fn call(- &self,- _ctx: Context,- _request: Request,- decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, LlmClientError> {- self.barrier.wait().await;- Ok(Response {- llm_response: LlmResponse::Agg(text_response(- None,- decision.selected_model().to_string(),- )),- metadata: None,- })- }- }-+ // Serving blocks until all N concurrent calls have arrived. If requests were+ // serialized (one algorithm behind a `Mutex`), only one call could be in flight,+ // the barrier would never reach N, and the test would time out. It passes only+ // because the shared algorithm is driven concurrently across requests.let barrier = Arc::new(Barrier::new(N));- let targets = LlmTargetSet::new(vec![LlmTarget {- semantic_name: "m".to_string(),- llm_client: Some(Arc::new(BarrierClient {- barrier: barrier.clone(),- })),- }]);// One shared algorithm driven by many concurrent requests.- let algo = orch(targets);+ let algo = orch(target_set(&["m"]));let mut handles = Vec::new();for _ in 0..N {let algo = algo.clone();+ let barrier = barrier.clone();+ let serve = move |decision: Arc<dyn Decision>, _request: Request| {+ let barrier = barrier.clone();+ async move {+ barrier.wait().await;+ Ok(reply(decision.selected_model()))+ }+ };handles.push(tokio::spawn(async move {- algo.run(Context::default(), request())+ drive(algo, Context::default(), request(), serve).await.map(|(_, response)| {response@@ -1341,11 +1128,8 @@ mod tests {// A client-less target offloads its call; we fulfill the promise with an// Err, which must flow back through `call_llm_target` into the algorithm and// out as an error step — not a response.- let stream = orch(target_set(&[("offload/model", false)])).run_stream(- Context::default(),- request(),- None,- );+ let stream =+ orch(target_set(&["offload/model"])).run_stream(Context::default(), request(), None);tokio::pin!(stream);let mut saw_error = false;@@ -1498,9 +1282,9 @@ mod tests {}let algo: Arc<dyn Algorithm> = Arc::new(Panicky);- match algo.run(Context::default(), request()).await {+ match drive(algo, Context::default(), request(), echo()).await {Ok(_) => Err(test_error(- "expected run to surface the algorithm panic as an error",+ "expected the run to surface the algorithm panic as an error",)),Err(err) => {assert!(matches!(err, LibsyError::AlgorithmTask { .. }));@@ -1557,9 +1341,10 @@ mod tests {dropped: dropped.clone(),});- // Drive `run` on its own task, wait until the algorithm task is up, then cancel- // `run` — dropping its future (and the `run_stream` stream it holds).- let run_task = tokio::spawn(async move { algo.run(Context::default(), request()).await });+ // Drive the run on its own task, wait until the algorithm task is up, then cancel+ // it — dropping its future (and the `run_stream` stream it holds).+ let run_task =+ tokio::spawn(async move { drive(algo, Context::default(), request(), echo()).await });started_rx.recv().await@@ -1576,62 +1361,6 @@ mod tests {// --- first-wins hedging: `run` must not wait on losing speculative calls -------------- /// The loser: signals it has started serving (so the winner can then win with the- /// loser's serve guaranteed in flight), then finishes late (`Some(delay)`) or never- /// (`None`).- struct LoserClient {- started: Arc<tokio::sync::Notify>,- delay: Option<std::time::Duration>,- }-- #[async_trait]- impl RoutedLlmClient for LoserClient {- async fn call(- &self,- _ctx: Context,- _request: Request,- decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, LlmClientError> {- self.started.notify_one();- match self.delay {- Some(delay) => tokio::time::sleep(delay).await,- None => std::future::pending::<()>().await,- }- Ok(Response {- llm_response: LlmResponse::Agg(text_response(- None,- decision.selected_model().to_string(),- )),- metadata: None,- })- }- }-- /// The winner: waits until the loser's serve has started, then echoes immediately, so- /// the loser's serve is guaranteed in flight when the winner wins.- struct GatedEchoClient {- gate: Arc<tokio::sync::Notify>,- }-- #[async_trait]- impl RoutedLlmClient for GatedEchoClient {- async fn call(- &self,- _ctx: Context,- _request: Request,- decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, LlmClientError> {- self.gate.notified().await;- Ok(Response {- llm_response: LlmResponse::Agg(text_response(- None,- decision.selected_model().to_string(),- )),- metadata: None,- })- }- }-/// Offloads two targets concurrently and returns the first to resolve, dropping the/// loser's call (first-wins hedging).struct Hedge {@@ -1657,8 +1386,16 @@ mod tests {let dec_l: Arc<dyn Decision> = Arc::new(TestDecision {model: self.loser.semantic_name.clone(),});- let win = driver.call_llm_target(ctx.clone(), &self.winner, request.clone(), dec_w);- let lose = driver.call_llm_target(ctx, &self.loser, request, dec_l);+ let win = driver.call_llm(RoutedRequest {+ request: request.clone(),+ decision: dec_w,+ ctx: ctx.clone(),+ });+ let lose = driver.call_llm(RoutedRequest {+ request,+ decision: dec_l,+ ctx,+ });// First to resolve wins; `select!` drops the losing future (and its promise).tokio::select! {res = win => res,@@ -1667,33 +1404,43 @@ mod tests {}}- /// Builds a hedging algo whose winner is gated behind the loser starting, and whose- /// loser finishes after `loser_delay` (or never, when `None`).- fn hedge(loser_delay: Option<std::time::Duration>) -> Arc<dyn Algorithm> {+ /// Builds a hedging algo and the `serve` that drives it: the winner is gated behind the+ /// loser starting (so the loser's serve is guaranteed in flight when the winner wins),+ /// and the loser finishes after `loser_delay` — or never, when `None`.+ fn hedge(loser_delay: Option<std::time::Duration>) -> (Arc<dyn Algorithm>, impl Serve) {let started = Arc::new(tokio::sync::Notify::new());- let winner = LlmTarget {- semantic_name: "winner".to_string(),- llm_client: Some(Arc::new(GatedEchoClient {- gate: started.clone(),- })),- };- let loser = LlmTarget {- semantic_name: "loser".to_string(),- llm_client: Some(Arc::new(LoserClient {- started,- delay: loser_delay,- })),+ let algo = Arc::new(Hedge {+ winner: LlmTarget {+ semantic_name: "winner".to_string(),+ },+ loser: LlmTarget {+ semantic_name: "loser".to_string(),+ },+ });+ let serve = move |decision: Arc<dyn Decision>, _request: Request| {+ let started = started.clone();+ async move {+ if decision.selected_model() == "loser" {+ started.notify_one();+ match loser_delay {+ Some(delay) => tokio::time::sleep(delay).await,+ None => std::future::pending::<()>().await,+ }+ } else {+ started.notified().await;+ }+ Ok(reply(decision.selected_model()))+ }};- Arc::new(Hedge { winner, loser })+ (algo, serve)}#[tokio::test]async fn run_returns_the_winner_without_a_late_loser_overwriting_it() -> Result<()> {// The loser responds 50ms after the winner has already won. `run` must return the// winner, not the loser's `respond`-to-a-dropped-receiver error.- let (_trace, response) = hedge(Some(std::time::Duration::from_millis(50)))- .run(Context::default(), request())- .await?;+ let (algo, serve) = hedge(Some(std::time::Duration::from_millis(50)));+ let (_trace, response) = drive(algo, Context::default(), request(), serve).await?;assert_eq!(response.llm_response@@ -1709,7 +1456,8 @@ mod tests {async fn run_returns_the_winner_without_hanging_on_a_pending_loser() -> Result<()> {// The loser never resolves. `run` must return the winner promptly, not hang// waiting for the in-flight loser.- let run = hedge(None).run(Context::default(), request());+ let (algo, serve) = hedge(None);+ let run = drive(algo, Context::default(), request(), serve);let (_trace, response) = tokio::time::timeout(std::time::Duration::from_secs(1), run).await.map_err(|error| LibsyError::external("waiting for pending loser", error))??;@@ -1732,33 +1480,9 @@ mod tests {// error must still reach the caller with all of these calls pending.const N: usize = 10;- // Enters each call; once all N are in flight, signals, then pends forever.- struct EnterThenPend {- started: Arc<AtomicUsize>,- all_started: Arc<tokio::sync::Notify>,- n: usize,- }-- #[async_trait]- impl RoutedLlmClient for EnterThenPend {- async fn call(- &self,- _ctx: Context,- _request: Request,- _decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, LlmClientError> {- if self.started.fetch_add(1, Ordering::SeqCst) + 1 == self.n {- self.all_started.notify_one();- }- std::future::pending::<()>().await;- unreachable!()- }- }-// Fans out N calls, then errors as soon as all N are in flight — exercising a// terminal failure emitted while the offloaded calls are still pending.struct FanOutThenError {- target: LlmTarget,all_started: Arc<tokio::sync::Notify>,n: usize,}@@ -1779,7 +1503,11 @@ mod tests {let decision: Arc<dyn Decision> = Arc::new(TestDecision {model: format!("m{i}"),});- driver.call_llm_target(ctx.clone(), &self.target, request.clone(), decision)+ driver.call_llm(RoutedRequest {+ request: request.clone(),+ decision,+ ctx: ctx.clone(),+ })}));tokio::select! {_ = offloads => Err(test_error("offloads unexpectedly completed")),@@ -1791,23 +1519,27 @@ mod tests {}let all_started = Arc::new(tokio::sync::Notify::new());- let target = LlmTarget {- semantic_name: "pending".to_string(),- llm_client: Some(Arc::new(EnterThenPend {- started: Arc::new(AtomicUsize::new(0)),- all_started: all_started.clone(),- n: N,- })),- };let algo: Arc<dyn Algorithm> = Arc::new(FanOutThenError {- target,- all_started,+ all_started: all_started.clone(),n: N,});- // With the cap gone, `run` keeps polling the stream even with N calls in flight, so- // the terminal error surfaces promptly instead of hanging.- let run = algo.run(Context::default(), request());+ // Serving enters each call; once all N are in flight it signals, then pends forever.+ let started = Arc::new(AtomicUsize::new(0));+ let serve = move |_decision: Arc<dyn Decision>, _request: Request| {+ let started = started.clone();+ let all_started = all_started.clone();+ async move {+ if started.fetch_add(1, Ordering::SeqCst) + 1 == N {+ all_started.notify_one();+ }+ std::future::pending::<ServeResult>().await+ }+ };++ // With the cap gone, the driver keeps polling the stream even with N calls in+ // flight, so the terminal error surfaces promptly instead of hanging.+ let run = drive(algo, Context::default(), request(), serve);let result = tokio::time::timeout(std::time::Duration::from_millis(500), run).await.map_err(|error| {diff --git a/crates/libsy/src/core/testing.rs b/crates/libsy/src/core/testing.rsnew file mode 100644index 000000000..9a0e4f7aa--- /dev/null+++ b/crates/libsy/src/core/testing.rs@@ -0,0 +1,102 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Fake clients for libsy's own tests.+//!+//! libsy offloads every model call, so a test needs something to answer them. [`drive`] is+//! [`crate::drive`] with the promise handling filled in: a test supplies a [`Serve`] closure+//! standing in for the client a real host would use, and gets back the same+//! `(trace, response)` pair a host does — the same path `switchyard-llm-client`'s `run`+//! takes over HTTP.+//!+//! The closure is async so a fake can block on a barrier, wait on a notify, or never+//! resolve, which is what the concurrency, hedging, and fan-out tests need.++use std::future::Future;+use std::sync::Arc;++use futures::future::BoxFuture;+use switchyard_protocol::{+ Context, Decision, LlmClientError, LlmResponse, Request, Response, text_response,+};++use crate::core::algorithm::{Algorithm, CallLlmRequest, RunObserver};+use crate::{LibsyError, Result};++/// The result a fake client hands back for one offloaded call.+pub(crate) type ServeResult = std::result::Result<Response, LlmClientError>;++/// Answers offloaded model calls. Returning `Err` propagates a failed *model* call back into+/// the algorithm, which may route around it.+pub(crate) trait Serve: Send + Sync + 'static {+ fn serve(+ &self,+ decision: Arc<dyn Decision>,+ request: Request,+ ) -> BoxFuture<'static, ServeResult>;+}++impl<F, Fut> Serve for F+where+ F: Fn(Arc<dyn Decision>, Request) -> Fut + Send + Sync + 'static,+ Fut: Future<Output = ServeResult> + Send + 'static,+{+ fn serve(+ &self,+ decision: Arc<dyn Decision>,+ request: Request,+ ) -> BoxFuture<'static, ServeResult> {+ Box::pin(self(decision, request))+ }+}++/// Run `algorithm` to completion, serving each offloaded call with `serve`.+pub(crate) async fn drive(+ algorithm: Arc<dyn Algorithm>,+ ctx: Context,+ request: Request,+ serve: impl Serve,+) -> Result<(Vec<Arc<dyn Decision>>, Response)> {+ drive_observed(algorithm, ctx, request, None, serve).await+}++/// [`drive`] plus a [`RunObserver`], for tests that assert on run observations.+pub(crate) async fn drive_observed(+ algorithm: Arc<dyn Algorithm>,+ ctx: Context,+ request: Request,+ observer: Option<RunObserver>,+ serve: impl Serve,+) -> Result<(Vec<Arc<dyn Decision>>, Response)> {+ let serve = Arc::new(serve);+ crate::drive(algorithm, ctx, request, observer, move |call| {+ fulfill(Arc::clone(&serve), call)+ })+ .await+}++/// Serve one call and fulfill its promise, mapping failures the way a host does so+/// error-shape assertions match production.+async fn fulfill(serve: Arc<impl Serve>, call: CallLlmRequest) -> Result<()> {+ let routed = call.get_routed().clone();+ let target = routed.decision.selected_model().to_string();+ let result = serve+ .serve(routed.decision, routed.request)+ .await+ .map_err(|source| LibsyError::client_call(target, source));+ call.respond(result)+}++/// Answers with the selected model name as the completion — what most routing tests need,+/// since they assert on *which* target was called.+pub(crate) fn echo() -> impl Serve {+ |decision: Arc<dyn Decision>, _request: Request| async move { Ok(reply(decision.selected_model())) }+}++/// A buffered response whose completion text is `completion`.+pub(crate) fn reply(completion: impl Into<String>) -> Response {+ Response {+ llm_response: LlmResponse::Agg(text_response(None, completion.into())),+ metadata: None,+ }+}diff --git a/crates/libsy/src/error.rs b/crates/libsy/src/error.rsindex 834ceff33..648245d82 100644--- a/crates/libsy/src/error.rs+++ b/crates/libsy/src/error.rs@@ -32,13 +32,6 @@ pub enum LibsyError {message: String,},- /// A routed target had no default client for [`crate::Algorithm::run`].- #[error("target {target:?} has no client to serve the call")]- MissingClient {- /// Target that could not be served.- target: String,- },-/// The type-erased offload driver could not complete an operation.#[error(transparent)]Driver(#[from] DriverError),diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rsindex 37f6cb22f..d6593d23f 100644--- a/crates/libsy/src/lib.rs+++ b/crates/libsy/src/lib.rs@@ -7,7 +7,7 @@mod core;pub use core::algorithm::{Algorithm, CallLlmRequest, Driver, LlmCallObservation, LlmTarget, LlmTargetSet, RoutedRequest,- RunObservation, RunObserver, Step, StepStream,+ RunObservation, RunObserver, Step, StepStream, drive,};pub use core::classifier::{Classification, Classifier, Score};pub use core::processor::{Event, Processor};@@ -41,6 +41,7 @@ pub use algorithms::util::stage::{};mod observability;+pub use observability::algorithm_label;/// Registers process-wide compatibility gauges with the global meter provider.///diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rsindex bf3f1f1fe..1e02bcc88 100644--- a/crates/libsy/src/observability.rs+++ b/crates/libsy/src/observability.rs@@ -4,9 +4,11 @@//! OpenTelemetry metrics plus `tracing` spans and structured logs for the//! algorithm layer.//!-//! The crate's provided run methods call these helpers around the [`Decision`]-//! hook and the offload boundary, so every algorithm is instrumented from the-//! outside and carries no telemetry code of its own. Metrics record through the+//! [`Algorithm::run_stream`](crate::Algorithm::run_stream) and [`Driver`] call these+//! helpers around the [`Decision`] hook and the offload boundary, so every algorithm is+//! instrumented from the outside and carries no telemetry code of its own. The provider+//! call on the other side of the offload belongs to the host, and is instrumented by+//! whoever makes it. Metrics record through the//! OpenTelemetry **global** meter provider under the `switchyard` scope — the host//! installs an SDK provider and exporters; with none installed, recording is a//! no-op. Spans and logs use the `tracing` facade (the async-native surface the@@ -30,26 +32,17 @@//! provider installed at any point in the process lifetime; the cost is//! negligible next to a model call.-use std::borrow::Cow;use std::future::Future;-use std::pin::Pin;use std::sync::OnceLock;use std::sync::atomic::{AtomicU64, Ordering};-use std::task::{Context as TaskContext, Poll};use std::time::{Duration, Instant};-use futures::Stream;use opentelemetry::metrics::{Meter, ObservableGauge};-use opentelemetry::{Array as OtelArray, KeyValue, StringValue, Value as OtelValue, global};-use switchyard_protocol::StopReason;+use opentelemetry::{KeyValue, global};use tracing::Span;-use tracing_opentelemetry::OpenTelemetrySpanExt;-use crate::{Driver, LibsyError, Result};-use switchyard_protocol::{- AggLlmResponse, Context, Decision, LlmClientError, LlmRequest, LlmResponse, LlmResponseChunk,- LlmResponseStream, LlmResponseStreamEvent, Request, Response, Usage,-};+use crate::{Driver, Result};+use switchyard_protocol::{Context, Decision, Request, Response};const METRICS_SCOPE: &str = "switchyard";const TRACING_TARGET: &str = "libsy";@@ -62,8 +55,12 @@ static TOTAL_GAUGES: OnceLock<(ObservableGauge<u64>, ObservableGauge<u64>)> = On/// telemetry label ([`Algorithm::name`](crate::Algorithm::name)).pub(crate) const ALGORITHM_KEY: &str = "algorithm";-/// The algorithm label carried by a request context; empty until stamped.-pub(crate) fn algorithm_label(ctx: &Context) -> &str {+/// The algorithm label [`Algorithm::run_stream`](crate::Algorithm::run_stream) stamps into+/// a request context; empty until stamped.+///+/// A host instrumenting the calls it serves reads this to attribute its own spans to the+/// algorithm that produced them.+pub fn algorithm_label(ctx: &Context) -> &str {ctx.values.get(ALGORITHM_KEY).map(String::as_str)@@ -171,295 +168,6 @@ pub(crate) async fn observe_run(result}-/// Records request parameters represented directly by the neutral IR.-pub(crate) fn record_gen_ai_request(span: &Span, request: &LlmRequest) {- if request.stream {- span.record("gen_ai.request.stream", true);- }- if let Some(value) = request.sampling.temperature {- span.record("gen_ai.request.temperature", value);- }- if let Some(value) = request.sampling.top_p {- span.record("gen_ai.request.top_p", value);- }- if let Some(value) = request.sampling.top_k {- span.record("gen_ai.request.top_k", value);- }- if let Some(value) = request.output.max_output_tokens {- span.record("gen_ai.request.max_tokens", otel_int(value));- }- if let Some(value) = request.reasoning.effort.as_deref() {- span.record("gen_ai.request.reasoning.level", value);- }- if let Some(value) = request- .output- .response_format- .as_ref()- .and_then(gen_ai_output_type)- {- span.record("gen_ai.output.type", value);- }-}--/// Adds terminal response and usage fields to the enclosing `libsy.client_call`-/// span without consuming or buffering a streaming response.-pub(crate) fn observe_client_call(result: Result<Response>) -> Result<Response> {- let span = Span::current();- match result {- Ok(mut response) => {- match response.llm_response {- LlmResponse::Agg(agg) => {- span.record("outcome", "ok");- record_gen_ai_response(&span, &agg);- response.llm_response = LlmResponse::Agg(agg);- }- LlmResponse::Stream(stream) => {- response.llm_response =- LlmResponse::Stream(observe_client_stream(stream, span));- }- }- Ok(response)- }- Err(error) => {- let error_type = client_call_error_type(&error);- record_client_error(&span, &error_type, &error);- Err(error)- }- }-}--fn observe_client_stream(stream: LlmResponseStream, span: Span) -> LlmResponseStream {- Box::pin(ObservedClientStream {- stream,- observer: Some(ClientStreamObserver {- span,- terminal: false,- }),- })-}--fn record_gen_ai_response(span: &Span, response: &AggLlmResponse) {- record_optional(span, "gen_ai.response.id", response.id.as_deref());- record_optional(span, "gen_ai.response.model", response.model.as_deref());- record_finish_reasons(- span,- response- .outputs- .iter()- .filter_map(|output| output.stop_reason)- .map(stop_reason_name),- );- record_gen_ai_usage(span, &response.usage);-}--fn record_gen_ai_usage(span: &Span, usage: &Usage) {- let cache_read = usage.cached_input_tokens();- let cache_creation = usage.cache_creation_input_tokens();- if usage.input_tokens.is_some() || cache_read.is_some() || cache_creation.is_some() {- let input_tokens = usage- .input_tokens- .unwrap_or_default()- .saturating_add(cache_read.unwrap_or_default())- .saturating_add(cache_creation.unwrap_or_default());- span.record("gen_ai.usage.input_tokens", otel_int(input_tokens));- }- for (field, value) in [- ("gen_ai.usage.output_tokens", usage.output_tokens),- ("gen_ai.usage.cache_read.input_tokens", cache_read),- ("gen_ai.usage.cache_creation.input_tokens", cache_creation),- (- "gen_ai.usage.reasoning.output_tokens",- usage.reasoning_tokens,- ),- ] {- if let Some(value) = value {- span.record(field, otel_int(value));- }- }-}--// OpenTelemetry integer attributes are signed; token counts are unsigned in the IR.-fn otel_int(value: u64) -> i64 {- value.min(i64::MAX as u64) as i64-}--fn record_optional(span: &Span, field: &str, value: Option<&str>) {- if let Some(value) = value {- span.record(field, value);- }-}--// `tracing` fields cannot preserve a typed string array, so write this attribute directly.-fn record_finish_reasons(span: &Span, reasons: impl IntoIterator<Item = impl Into<String>>) {- let reasons = reasons- .into_iter()- .map(|reason| StringValue::from(reason.into()))- .collect::<Vec<_>>();- if !reasons.is_empty() {- span.set_attribute(- "gen_ai.response.finish_reasons",- OtelValue::Array(OtelArray::String(reasons)),- );- }-}--fn stop_reason_name(reason: StopReason) -> &'static str {- match reason {- StopReason::EndTurn => "end_turn",- StopReason::MaxTokens => "max_tokens",- StopReason::ToolUse => "tool_use",- StopReason::ContentFilter => "content_filter",- StopReason::Error => "error",- StopReason::Unknown => "unknown",- }-}--fn gen_ai_output_type(response_format: &serde_json::Value) -> Option<&'static str> {- match response_format- .get("type")- .and_then(serde_json::Value::as_str)- {- Some("json" | "json_object" | "json_schema") => Some("json"),- Some("text") => Some("text"),- _ => None,- }-}--fn client_call_error_type(error: &LibsyError) -> Cow<'static, str> {- match error {- LibsyError::ClientCall { source, .. } => llm_client_error_type(source),- LibsyError::TargetNotFound { .. } => Cow::Borrowed("target_not_found"),- LibsyError::NoTargets => Cow::Borrowed("no_targets"),- LibsyError::AlgorithmError { .. } => Cow::Borrowed("algorithm_error"),- LibsyError::MissingClient { .. } => Cow::Borrowed("client_configuration_error"),- LibsyError::Driver(_) => Cow::Borrowed("driver_error"),- LibsyError::AlgorithmTask { .. } => Cow::Borrowed("algorithm_task_error"),- LibsyError::MissingFinalResponse => Cow::Borrowed("missing_final_response"),- LibsyError::AllTargetsExcluded => Cow::Borrowed("context_window_exceeded"),- LibsyError::External { .. } => Cow::Borrowed("_OTHER"),- }-}--fn llm_client_error_type(error: &LlmClientError) -> Cow<'static, str> {- match error {- LlmClientError::InvalidRequest { .. } => Cow::Borrowed("invalid_request"),- LlmClientError::RequestTranslation(_) => Cow::Borrowed("request_translation"),- LlmClientError::RequestEncoding(_) => Cow::Borrowed("request_encoding"),- LlmClientError::ResponseTranslation(_) => Cow::Borrowed("response_translation"),- LlmClientError::Configuration { .. } => Cow::Borrowed("configuration"),- LlmClientError::Transport { .. } => Cow::Borrowed("transport"),- LlmClientError::Timeout { .. } => Cow::Borrowed("timeout"),- LlmClientError::ContextWindowExceeded { .. } => Cow::Borrowed("context_window_exceeded"),- LlmClientError::UpstreamHttp { status, .. } => Cow::Owned(status.to_string()),- LlmClientError::InvalidResponse { .. } => Cow::Borrowed("invalid_response"),- LlmClientError::Ffi { .. } => Cow::Borrowed("ffi"),- _ => Cow::Borrowed("_OTHER"),- }-}--// Keep the client span alive until the response is drained, errors, or is abandoned.-struct ObservedClientStream {- stream: LlmResponseStream,- observer: Option<ClientStreamObserver>,-}--impl Stream for ObservedClientStream {- type Item = std::result::Result<LlmResponseStreamEvent, LlmClientError>;-- fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Option<Self::Item>> {- match self.stream.as_mut().poll_next(cx) {- Poll::Ready(Some(item)) => {- let failed = self- .observer- .as_mut()- .is_some_and(|observer| observer.observe(&item));- if failed {- self.observer.take();- }- Poll::Ready(Some(item))- }- Poll::Ready(None) => {- if let Some(mut observer) = self.observer.take() {- observer.complete();- }- Poll::Ready(None)- }- Poll::Pending => Poll::Pending,- }- }-}--struct ClientStreamObserver {- span: Span,- terminal: bool,-}--impl ClientStreamObserver {- fn observe(- &mut self,- item: &std::result::Result<LlmResponseStreamEvent, LlmClientError>,- ) -> bool {- match item {- Ok(event) => {- for chunk in event.normalized() {- self.observe_chunk(chunk);- if self.terminal {- break;- }- }- }- Err(error) => {- let error_type = llm_client_error_type(error);- record_client_error(&self.span, &error_type, error);- self.terminal = true;- }- }- self.terminal- }-- fn observe_chunk(&mut self, chunk: &LlmResponseChunk) {- match chunk {- LlmResponseChunk::MessageStart { id, model } => {- record_optional(&self.span, "gen_ai.response.id", id.as_deref());- record_optional(&self.span, "gen_ai.response.model", model.as_deref());- }- LlmResponseChunk::Usage(usage) => record_gen_ai_usage(&self.span, usage),- LlmResponseChunk::MessageStop { reason } => {- record_finish_reasons(&self.span, reason.iter().cloned());- }- LlmResponseChunk::DecodeError { message } => {- record_client_error(&self.span, "response_translation", message);- self.terminal = true;- }- LlmResponseChunk::StreamError { message } => {- record_client_error(&self.span, "502", message);- self.terminal = true;- }- _ => {}- }- }-- fn complete(&mut self) {- self.span.record("outcome", "ok");- self.terminal = true;- }-}--impl Drop for ClientStreamObserver {- fn drop(&mut self) {- if !self.terminal {- self.span.record("outcome", "cancelled");- }- }-}--fn record_client_error(span: &Span, error_type: &str, error: &dyn std::fmt::Display) {- span.record("outcome", "error");- span.record("otel.status_code", "ERROR");- span.record("error.type", error_type);- span.record("error", tracing::field::display(error));-}-/// Records the end of one algorithm run: the run counter and duration/// histogram, the `outcome`/`error` fields on `span`, and a warn log when the/// run failed.diff --git a/crates/switchyard-py/Cargo.toml b/crates/switchyard-py/Cargo.tomlindex 378e1517a..ceaa7cf7f 100644--- a/crates/switchyard-py/Cargo.toml+++ b/crates/switchyard-py/Cargo.toml@@ -21,6 +21,7 @@ futures-util.workspace = truehttp.workspace = trueparking_lot.workspace = trueswitchyard-libsy.workspace = true+switchyard-llm-client.workspace = truepyo3 = { version = "0.28.3", features = ["abi3-py312", "extension-module"] }pyo3-async-runtimes = { version = "0.28", features = ["tokio-runtime"] }pythonize = "0.28.0"diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rsindex 7f13746ad..d33934706 100644--- a/crates/switchyard-py/src/libsy_bindings.rs+++ b/crates/switchyard-py/src/libsy_bindings.rs@@ -14,6 +14,7 @@ use switchyard_libsy::{LlmClassifierConfig, LlmFallback, LlmTarget, LlmTargetSet, LlmTaskClassifier, Noop, PickerMode,Random, StageRouter, StageRouterConfig, TaskClassifierConfig,};+use switchyard_llm_client::ClientRouter;use switchyard_protocol::{AggLlmResponse, Context, Decision, LlmClientError, LlmResponse, Metadata, Request, Response,RoutedLlmClient,@@ -62,14 +63,23 @@ struct PyLlmTarget {}impl PyLlmTarget {- fn clone_core(&self, py: Python<'_>) -> LlmTarget {+ fn clone_core(&self, _py: Python<'_>) -> LlmTarget {LlmTarget {semantic_name: self.name.clone(),- llm_client: Some(Arc::new(PythonLlmClient {- inner: self.client.clone_ref(py),- })),}}++ /// The `selected_model -> client` entry this target contributes to the algorithm's+ /// [`ClientRouter`]. libsy no longer carries the client, so the bindings keep the+ /// mapping and serve the calls themselves.+ fn client_entry(&self, py: Python<'_>) -> ClientEntry {+ (+ self.name.clone(),+ Arc::new(PythonLlmClient {+ inner: self.client.clone_ref(py),+ }),+ )+ }}#[pymethods]@@ -170,6 +180,11 @@ struct PyLlmFallback {}impl PyLlmFallback {+ /// The judge's client entry, so the caller can register it with the algorithm's router.+ fn judge_client_entry(&self, py: Python<'_>) -> PyResult<ClientEntry> {+ Ok(self.judge_target.bind(py).try_borrow()?.client_entry(py))+ }+fn clone_core(&self, py: Python<'_>) -> PyResult<LlmFallback> {Ok(LlmFallback {judge_target: self.judge_target.bind(py).try_borrow()?.clone_core(py),@@ -194,11 +209,19 @@ impl PyLlmFallback {#[pyclass(name = "Algorithm", module = "switchyard.libsy", frozen)]struct PyAlgorithm {inner: Arc<dyn Algorithm>,+ /// Resolves the calls `inner` offloads to each target's Python client.+ client_router: ClientRouter,}+/// One target's `selected_model -> client` mapping for an algorithm's router.+type ClientEntry = (String, Arc<dyn RoutedLlmClient>);+impl PyAlgorithm {- fn new(inner: Arc<dyn Algorithm>) -> Self {- Self { inner }+ fn new(inner: Arc<dyn Algorithm>, clients: impl IntoIterator<Item = ClientEntry>) -> Self {+ Self {+ inner,+ client_router: clients.into_iter().collect(),+ }}}@@ -218,6 +241,7 @@ impl PyAlgorithm {headers: Option<std::collections::HashMap<String, String>>,) -> PyResult<Bound<'py, PyAny>> {let algorithm = Arc::clone(&self.inner);+ let client_router = self.client_router.clone();let headers = headers.as_ref().map(header_map_from_python).transpose()?;let request = Request {@@ -226,10 +250,15 @@ impl PyAlgorithm {metadata: headers.map(|headers| Metadata::from_headers(&headers)),};pyo3_async_runtimes::tokio::future_into_py(py, async move {- let (decisions, response) = algorithm- .run(Context::default(), request)- .await- .map_err(py_libsy_error)?;+ let (decisions, response) = switchyard_llm_client::run(+ algorithm,+ client_router,+ Context::default(),+ request,+ None,+ )+ .await+ .map_err(py_libsy_error)?;let response = response.llm_response.into_agg()@@ -256,7 +285,8 @@ impl PyAlgorithm {/// Construct the no-op reference algorithm.#[pyfunction(name = "noop")]fn noop_algorithm() -> PyAlgorithm {- PyAlgorithm::new(Arc::new(Noop {}))+ // `Noop` synthesizes its own response and never offloads a call, so it needs no clients.+ PyAlgorithm::new(Arc::new(Noop {}), [])}/// Construct random routing over targets with optional relative weights and seed.@@ -268,18 +298,31 @@ fn random_algorithm(weights: Option<Vec<f64>>,seed: Option<u64>,) -> PyResult<PyAlgorithm> {- let targets = targets- .iter()- .map(|target| Ok(target.bind(py).try_borrow()?.clone_core(py)))- .collect::<PyResult<Vec<_>>>()?;+ let (cores, clients) = target_cores(py, &targets)?;let algorithm =- Random::new(LlmTargetSet::new(targets), weights, seed).map_err(|error| match error {+ Random::new(LlmTargetSet::new(cores), weights, seed).map_err(|error| match error {RustLibsyError::NoTargets => {PyValueError::new_err("random requires at least one target")}other => PyValueError::new_err(other.to_string()),})?;- Ok(PyAlgorithm::new(Arc::new(algorithm)))+ Ok(PyAlgorithm::new(Arc::new(algorithm), clients))+}++/// Splits a list of Python targets into libsy's client-free targets and the client entries+/// the bindings keep for the algorithm's router.+fn target_cores(+ py: Python<'_>,+ targets: &[Py<PyLlmTarget>],+) -> PyResult<(Vec<LlmTarget>, Vec<ClientEntry>)> {+ let mut cores = Vec::with_capacity(targets.len());+ let mut clients = Vec::with_capacity(targets.len());+ for target in targets {+ let target = target.bind(py).try_borrow()?;+ cores.push(target.clone_core(py));+ clients.push(target.client_entry(py));+ }+ Ok((cores, clients))}/// Construct task-level LLM classifier routing.@@ -298,14 +341,21 @@ fn llm_task_classifier_algorithm(capable_target: Py<PyLlmTarget>,config: Py<PyTaskClassifierConfig>,) -> PyResult<PyAlgorithm> {+ let (cores, clients) = target_cores(+ py,+ &[judge_target.clone_ref(py), efficient_target, capable_target],+ )?;+ let [judge, efficient, capable] = cores+ .try_into()+ .map_err(|_| PyValueError::new_err("expected three targets"))?;let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Capability {- judge_target: judge_target.bind(py).try_borrow()?.clone_core(py),- efficient_target: efficient_target.bind(py).try_borrow()?.clone_core(py),- capable_target: capable_target.bind(py).try_borrow()?.clone_core(py),+ judge_target: judge,+ efficient_target: efficient,+ capable_target: capable,config: config.bind(py).try_borrow()?.clone_core(),}).map_err(|error| PyValueError::new_err(error.to_string()))?;- Ok(PyAlgorithm::new(Arc::new(algorithm)))+ Ok(PyAlgorithm::new(Arc::new(algorithm), clients))}/// Construct signal-driven stage routing with an optional LLM classifier fallback.@@ -348,8 +398,10 @@ fn stage_router_algorithm()));}};- let capable = capable_target.bind(py).try_borrow()?.clone_core(py);- let efficient = efficient_target.bind(py).try_borrow()?.clone_core(py);+ let (cores, mut clients) = target_cores(py, &[capable_target, efficient_target])?;+ let [capable, efficient] = cores+ .try_into()+ .map_err(|_| PyValueError::new_err("expected two targets"))?;let mut config = StageRouterConfig::new(mode, confidence_threshold);config.recent_window = recent_window;config.handoff_notes = match (escalation_note, deescalation_note) {@@ -375,13 +427,18 @@ fn stage_router_algorithm(.tier_prompts.with(efficient.semantic_name.clone(), prompt);}+ // The judge is only reachable through the optional classifier fallback, so its client+ // joins the router only when a fallback is configured.+ if let Some(classifier) = &classifier {+ clients.push(classifier.bind(py).try_borrow()?.judge_client_entry(py)?);+ }config.llm_fallback = classifier.map(|classifier| classifier.bind(py).try_borrow()?.clone_core(py)).transpose()?;let algorithm = StageRouter::new(capable, efficient, config).map_err(|error| PyValueError::new_err(error.to_string()))?;- Ok(PyAlgorithm::new(Arc::new(algorithm)))+ Ok(PyAlgorithm::new(Arc::new(algorithm), clients))}fn other_python_error(error: PyErr) -> LlmClientError {diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.tomlindex 15391a98b..5d8a44939 100644--- a/crates/switchyard-server/Cargo.toml+++ b/crates/switchyard-server/Cargo.toml@@ -36,8 +36,8 @@ switchyard-translation.workspace = truehumantime = "2.4"tokio.workspace = truetracing.workspace = true-tracing-opentelemetry = "0.33"-tracing-subscriber = { version = "0.3", features = ["env-filter"] }+tracing-opentelemetry.workspace = true+tracing-subscriber.workspace = true[dev-dependencies]async-trait.workspace = truediff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rsindex 676efdf25..b644d4a04 100644--- a/crates/switchyard-server/src/config.rs+++ b/crates/switchyard-server/src/config.rs@@ -17,7 +17,8 @@ use libsy::{use serde::Deserialize;use serde_json::Value;use switchyard_llm_client::{- Backend, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig, TranslatingLlmClient,+ Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig,+ TranslatingLlmClient,};use switchyard_protocol::RoutedLlmClient;@@ -86,7 +87,7 @@ impl ServerConfig {}let clients = self.build_clients()?;- let targets = self.build_targets(&clients)?;+ let targets = self.build_targets()?;let mut routes = Vec::with_capacity(self.routes.len());for (route_name, config) in &self.routes {validate_value("route name", route_name)?;@@ -98,10 +99,12 @@ impl ServerConfig {)));}let algorithm = build_algorithm(route_name, config, &targets)?;+ let client = self.build_client_router(config, &clients)?;let count_tokens_target = self.build_count_tokens_target(config, &clients);routes.push((config.id().to_string(),algorithm,+ client,capabilities,count_tokens_target,));@@ -147,26 +150,51 @@ impl ServerConfig {Ok(clients)}- fn build_targets(- &self,- clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,- ) -> ServerResult<BTreeMap<String, LlmTarget>> {- self.targets+ fn build_targets(&self) -> ServerResult<BTreeMap<String, LlmTarget>> {+ Ok(self+ .targets.iter().map(|(name, config)| {- let client = clients.get(&config.llm_client).ok_or_else(|| {- ServerError::new(format!("target {name} has no constructed llm client"))- })?;- let client: Arc<dyn RoutedLlmClient> = client.clone();- Ok((+ (name.clone(),LlmTarget {semantic_name: config.id.clone(),- llm_client: Some(client),},- ))+ )})- .collect()+ .collect())+ }++ /// Maps every model this route can select to the client configured for that target.+ ///+ /// A route is a synthetic model (`switchyard/random`) with no upstream of its own — the+ /// algorithm picks a real target, and the *target* names the `[llm_clients.*]` section+ /// that serves it. Two targets in one route may therefore sit on different clients, and+ /// one request can emit calls for both.+ ///+ /// The map is built per route because targets are only reachable through the routes that+ /// name them, so the same model id may resolve to different clients in two routes without+ /// colliding.+ fn build_client_router(+ &self,+ route: &RouteConfig,+ clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,+ ) -> ServerResult<ClientRouter> {+ let by_model = route+ .callable_target_names()+ .into_iter()+ .map(|name| {+ let target = self.targets.get(name).ok_or_else(|| {+ ServerError::new(format!("route references unknown target {name}"))+ })?;+ let client = clients.get(&target.llm_client).ok_or_else(|| {+ ServerError::new(format!("target {name} has no constructed llm client"))+ })?;+ let client: Arc<dyn RoutedLlmClient> = client.clone();+ Ok((target.id.clone(), client))+ })+ .collect::<ServerResult<_>>()?;+ Ok(ClientRouter::new(by_model))}fn build_count_tokens_target(@@ -490,6 +518,25 @@ impl RouteConfig {}}+ /// Every target the algorithm may call, including judge-only targets.+ ///+ /// [`routing_target_names`](Self::routing_target_names) covers completion destinations;+ /// a classifier also calls its judge, and that call needs a client too.+ fn callable_target_names(&self) -> Vec<&str> {+ let mut names = self.routing_target_names();+ match self {+ Self::LlmClassifier {+ classifier_target, ..+ } => names.push(classifier_target),+ Self::StageRouter {+ classifier: Some(classifier),+ ..+ } => names.push(&classifier.target),+ _ => {}+ }+ names+ }+fn capabilities(&self) -> ModelCapabilities {use RouteConfig::*;match self {diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex 8f5c03461..4e4b28993 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -36,7 +36,7 @@ use libsy::{Algorithm, LibsyError, RunObservation, RunObserver};use parking_lot::Mutex;use serde::Deserialize;use serde_json::{Value, json};-use switchyard_llm_client::TranslatingLlmClient;+use switchyard_llm_client::{ClientRouter, TranslatingLlmClient};use switchyard_protocol::{Context, Decision, LlmClientError, Metadata, Request, Usage};use tokio::net::{TcpListener, TcpSocket};use tokio::task;@@ -106,6 +106,10 @@ struct ModelCapabilities {/// A registered algorithm route and its server-owned endpoint metadata.struct RouteEntry {algorithm: Arc<dyn Algorithm>,+ /// Resolves each offloaded call to the client configured for the target the algorithm+ /// selected. A route is a synthetic model with no upstream of its own, so this is a+ /// per-target lookup, never one client serving the whole route.+ target_clients: ClientRouter,capabilities: ModelCapabilities,count_tokens_target: Option<CountTokensTarget>,}@@ -168,15 +172,20 @@ impl SharedRoutingLog {}impl ServerState {- /// Creates server state from route model IDs and their libsy algorithms.+ /// Creates server state from route model IDs, their libsy algorithms, and the+ /// per-target client routing each route's calls are resolved through.pub fn new(- routes: impl IntoIterator<Item = (String, Arc<dyn Algorithm>)>,+ routes: impl IntoIterator<Item = (String, Arc<dyn Algorithm>, ClientRouter)>,) -> ServerResult<Self> {- Self::new_with_capabilities(- routes- .into_iter()- .map(|(model, algorithm)| (model, algorithm, ModelCapabilities::default(), None)),- )+ Self::new_with_capabilities(routes.into_iter().map(|(model, algorithm, clients)| {+ (+ model,+ algorithm,+ clients,+ ModelCapabilities::default(),+ None,+ )+ }))}fn new_with_capabilities(@@ -184,19 +193,21 @@ impl ServerState {Item = (String,Arc<dyn Algorithm>,+ ClientRouter,ModelCapabilities,Option<CountTokensTarget>,),>,) -> ServerResult<Self> {let mut entries = BTreeMap::new();- for (model, algorithm, capabilities, count_tokens_target) in routes {+ for (model, algorithm, target_clients, capabilities, count_tokens_target) in routes {let model = model.trim();if model.is_empty() {return Err(ServerError::new("route model must not be empty"));}let entry = RouteEntry {algorithm,+ target_clients,capabilities,count_tokens_target,};@@ -678,13 +689,21 @@ async fn handle_llm_request(Err(response) => return response,};let algorithm = Arc::clone(&route.algorithm);+ // Not one client for the route: a router that resolves each offloaded call to the+ // client configured for the target the algorithm selected.+ let target_clients = route.target_clients.clone();let observer = stats_observer(state.stats.clone(),state.routing_log.clone().zip(routing_log_context.clone()),);- let (trace, response) = match algorithm- .run_observed(Context::default(), request, Some(observer))- .await+ let (trace, response) = match switchyard_llm_client::run(+ algorithm,+ target_clients,+ Context::default(),+ request,+ Some(observer),+ )+ .await{Ok(result) => result,Err(error) => return algorithm_error(error),diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex 3f3907558..93f11f986 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -19,7 +19,9 @@ use axum::{Json, Router};use http_body_util::BodyExt;use libsy::{Algorithm, LlmTarget, LlmTargetSet, Random};use serde_json::{Value, json};-use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};+use switchyard_llm_client::{+ Backend, ClientRouter, HttpBackendConfig, ModelConfig, TranslatingLlmClient,+};use switchyard_protocol::RoutedLlmClient;use switchyard_server::config::load_server_state;use switchyard_server::{ServerState, build_switchyard_router};@@ -61,6 +63,16 @@ impl MockUpstream {task,})}++ /// The upstream model id of every request this upstream received, in order.+ async fn models(&self) -> Vec<String> {+ self.calls+ .lock()+ .await+ .iter()+ .filter_map(|call| call.get("model")?.as_str().map(str::to_string))+ .collect()+ }}impl Drop for MockUpstream {@@ -220,12 +232,15 @@ fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<Server.iter().map(|model| LlmTarget {semantic_name: (*model).to_string(),- llm_client: Some(Arc::clone(&client)),}).collect(),);let algorithm: Arc<dyn Algorithm> = Arc::new(Random::new(target_set, None, None)?);- Ok(((*route_model).to_string(), algorithm))+ Ok((+ (*route_model).to_string(),+ algorithm,+ ClientRouter::single(Arc::clone(&client)),+ ))}).collect::<TestResult<Vec<_>>>()?;Ok(ServerState::new(entries)?)@@ -647,6 +662,84 @@ fn assert_in_order(haystack: &str, needles: &[&str]) {}}+/// A route is a synthetic model (`switchyard/classify`) with no upstream of its own; its+/// algorithm picks real targets, and *those* name the client. One request can emit several+/// `Step::CallLlm` for different targets, and two targets may sit on different+/// `[llm_clients.*]` sections — here the judge is on one provider and the serving models on+/// another. Pin that each call reaches its own target's upstream, rather than one client+/// chosen per route serving all of them.+#[tokio::test]+async fn each_target_in_one_request_is_served_by_its_own_client() -> TestResult {+ let judge_upstream = MockUpstream::start().await?;+ let model_upstream = MockUpstream::start().await?;+ let state = load_test_config(&format!(+ r#"+schema_version = 1++[llm_clients.judge_provider]+format = "openai_chat"+base_url = "{judge_url}"++[llm_clients.model_provider]+format = "openai_chat"+base_url = "{model_url}"++[targets.judge]+id = "model/judge"+llm_client = "judge_provider"++[targets.strong]+id = "model/strong"+llm_client = "model_provider"++[targets.weak]+id = "model/weak"+llm_client = "model_provider"++[routes.classify]+id = "switchyard/classify"+type = "llm_classifier"+classifier_target = "judge"+strong_target = "strong"+weak_target = "weak"+base_threshold = 0.5+"#,+ judge_url = judge_upstream.base_url,+ model_url = model_upstream.base_url,+ ))?;+ let app = build_switchyard_router(state);++ let response = send(+ &app,+ "POST",+ "/v1/chat/completions",+ Some(json!({+ "model": "switchyard/classify",+ "messages": [{"role": "user", "content": "hi"}]+ })),+ )+ .await?;+ assert_eq!(response.status, StatusCode::OK);++ // The judge call went to the judge's provider and nowhere else; the serving call went to+ // the models' provider. A single per-route client would have sent both to one upstream.+ assert_eq!(+ judge_upstream.models().await,+ vec!["model/judge".to_string()]+ );+ let served = model_upstream.models().await;+ assert_eq!(+ served.len(),+ 1,+ "expected exactly one routed call: {served:?}"+ );+ assert!(+ served[0] == "model/weak" || served[0] == "model/strong",+ "routed call went to {served:?}"+ );+ Ok(())+}+/// A critical tool error must reach the stage router's signal scorer, which reads/// the decoded conversation. The endpoint records no inbound wire format, so a/// scorer that parsed the raw body instead would find nothing and route every turn