Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

chore: Collapse `CallModelRequest`, remove `Context` (#341)

Graham Kingfb3fc308ff13
diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.md
index e17655b9d..ed9d99188 100644
--- a/crates/libsy-llm-client/README.md
+++ b/crates/libsy-llm-client/README.md
@@ -80,7 +80,7 @@ fn build_client() -> switchyard_llm_client::Result<TranslatingLlmClient> {
```rust
use switchyard_llm_client::{LlmClientError, TranslatingLlmClient};
-use switchyard_protocol::{completion_text, text_request, Context, LlmResponse, Request};
+use switchyard_protocol::{completion_text, text_request, LlmResponse, Request};
async fn ask(client: &TranslatingLlmClient) -> switchyard_llm_client::Result<String> {
let request = Request {
@@ -91,7 +91,7 @@ async fn ask(client: &TranslatingLlmClient) -> switchyard_llm_client::Result<Str
// model_name wins over request.llm_request.model; it is also sent upstream.
let response = client
- .call_rewrite_model(Context::default(), request, Some("gpt-4o-mini"))
+ .call_rewrite_model(request, Some("gpt-4o-mini"))
.await?;
match response.llm_response {
@@ -110,7 +110,7 @@ Set `stream` on the IR request and drive the returned chunk stream:
```rust
use futures_util::StreamExt;
use switchyard_llm_client::TranslatingLlmClient;
-use switchyard_protocol::{text_request, Context, LlmResponse, LlmResponseChunk, Request};
+use switchyard_protocol::{text_request, LlmResponse, LlmResponseChunk, Request};
async fn stream(
client: &TranslatingLlmClient,
@@ -120,7 +120,7 @@ async fn stream(
let request = Request { llm_request, raw_request: None, metadata: None };
let response = client
- .call_rewrite_model(Context::default(), request, Some("gpt-4o-mini"))
+ .call_rewrite_model(request, Some("gpt-4o-mini"))
.await?;
if let LlmResponse::Stream(mut chunks) = response.llm_response {
@@ -147,7 +147,7 @@ single-provider case:
use std::sync::Arc;
use switchyard_libsy::Algorithm;
use switchyard_llm_client::{ClientRouter, TranslatingLlmClient};
-use switchyard_protocol::{Context, Request};
+use switchyard_protocol::Request;
async fn route(
algorithm: Arc<dyn Algorithm>,
@@ -156,7 +156,7 @@ async fn route(
) -> switchyard_libsy::Result<String> {
let clients = ClientRouter::single(client);
let (trace, _response) =
- switchyard_llm_client::run(algorithm, clients, Context::default(), request, None).await?;
+ switchyard_llm_client::run(algorithm, clients, request, None).await?;
Ok(trace
.last()
.map(|decision| decision.selected_model_id().to_string())
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index 0bede8d71..08ff14497 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -13,7 +13,7 @@ use reqwest::RequestBuilder;
use reqwest::header::{HeaderMap, RETRY_AFTER};
use serde_json::{Map, Value};
use switchyard_protocol::{
- Context, Decision, LlmRequest, LlmResponse, Metadata, Request, Response, RoutedLlmClient,
+ Decision, LlmRequest, LlmResponse, Metadata, Request, Response, RoutedLlmClient,
};
use switchyard_translation::{
WireFormat, decode_aggregated_response, decode_request, decode_stream,
@@ -362,7 +362,6 @@ impl TranslatingLlmClient {
/// models or wire formats are configuration errors.
pub async fn call_rewrite_model(
&self,
- _ctx: Context,
request: Request,
model_name: Option<&str>,
) -> Result<Response> {
@@ -459,7 +458,6 @@ impl TranslatingLlmClient {
/// set); pass `None` to forward nothing.
pub async fn call_rewrite_model_raw(
&self,
- ctx: Context,
raw_http_request: Value,
http_headers: Option<http::HeaderMap>,
model: Option<&str>,
@@ -488,7 +486,7 @@ impl TranslatingLlmClient {
..Default::default()
}),
};
- let response = self.call_rewrite_model(ctx, request, model).await?;
+ let response = self.call_rewrite_model(request, model).await?;
match response.llm_response {
LlmResponse::Agg(agg) => {
@@ -507,14 +505,9 @@ impl TranslatingLlmClient {
#[async_trait]
impl RoutedLlmClient for TranslatingLlmClient {
- async fn call(
- &self,
- ctx: Context,
- request: Request,
- decision: std::sync::Arc<Decision>,
- ) -> Result<Response> {
+ async fn call(&self, request: Request, decision: Decision) -> Result<Response> {
let model_name = Some(decision.selected_model_id());
- self.call_rewrite_model(ctx, request, model_name).await
+ self.call_rewrite_model(request, model_name).await
}
}
@@ -970,7 +963,7 @@ mod tests {
-> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
let client = TranslatingLlmClient::new(&[])?;
let Err(error) = client
- .call_rewrite_model(Context::default(), request_for(None, false), None)
+ .call_rewrite_model(request_for(None, false), None)
.await
else {
panic!("expected an error");
@@ -987,7 +980,7 @@ mod tests {
-> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
let client = TranslatingLlmClient::new(&[])?;
let Err(error) = client
- .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .call_rewrite_model(request_for(Some("gpt"), false), None)
.await
else {
panic!("expected an error");
@@ -1007,7 +1000,6 @@ mod tests {
let client = TranslatingLlmClient::new(&chat_map("https://example.test/v1"))?;
let Err(error) = client
.call_rewrite_model(
- Context::default(),
request_with_wire_format("gpt", WireFormat::AnthropicMessages),
None,
)
@@ -1049,7 +1041,7 @@ mod tests {
let client = TranslatingLlmClient::new(&[])?;
// Arg "b" is looked up (and reported), not the request's "a".
let Err(error) = client
- .call_rewrite_model(Context::default(), request_for(Some("a"), false), Some("b"))
+ .call_rewrite_model(request_for(Some("a"), false), Some("b"))
.await
else {
panic!("expected an error");
@@ -1084,7 +1076,7 @@ mod tests {
let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
let response = client
- .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .call_rewrite_model(request_for(Some("gpt"), false), None)
.await?;
let agg = response.llm_response.into_agg().await?;
assert_eq!(completion_text(&agg), "Hi there");
@@ -1109,7 +1101,7 @@ mod tests {
let client =
TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 2))?;
let Err(error) = client
- .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .call_rewrite_model(request_for(Some("gpt"), false), None)
.await
else {
panic!("expected invalid JSON to fail");
@@ -1130,7 +1122,7 @@ mod tests {
let (base_url, server) = truncated_response_server("application/json", "{}")?;
let client = TranslatingLlmClient::new(&chat_map(&base_url))?;
let result = client
- .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .call_rewrite_model(request_for(Some("gpt"), false), None)
.await;
server
.join()
@@ -1168,7 +1160,7 @@ mod tests {
response_sequence_server(vec![truncated.to_string(), raw_chat_success_response()])?;
let client = TranslatingLlmClient::new(&chat_map_with_retries(&base_url, 1))?;
let response = client
- .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .call_rewrite_model(request_for(Some("gpt"), false), None)
.await?;
server
.join()
@@ -1189,7 +1181,7 @@ mod tests {
let (base_url, server) = truncated_response_server("text/event-stream", body)?;
let client = TranslatingLlmClient::new(&chat_map_with_retries(&base_url, 2))?;
let response = client
- .call_rewrite_model(Context::default(), request_for(Some("gpt"), true), None)
+ .call_rewrite_model(request_for(Some("gpt"), true), None)
.await?;
let result = response.llm_response.into_agg().await;
server
@@ -1223,11 +1215,7 @@ mod tests {
let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
// Inbound model differs from the map key / resolved model.
client
- .call_rewrite_model(
- Context::default(),
- request_for(Some("switchyard"), false),
- Some("gpt"),
- )
+ .call_rewrite_model(request_for(Some("switchyard"), false), Some("gpt"))
.await?;
// The body_partial_json matcher asserts the upstream saw model "gpt".
Ok(())
@@ -1272,13 +1260,7 @@ mod tests {
});
client
- .call_rewrite_model_raw(
- Context::default(),
- raw,
- None,
- Some("gpt"),
- WireFormat::OpenAiChat,
- )
+ .call_rewrite_model_raw(raw, None, Some("gpt"), WireFormat::OpenAiChat)
.await?;
Ok(())
}
@@ -1347,13 +1329,7 @@ mod tests {
});
client
- .call_rewrite_model_raw(
- Context::default(),
- raw,
- None,
- Some("claude"),
- WireFormat::AnthropicMessages,
- )
+ .call_rewrite_model_raw(raw, None, Some("claude"), WireFormat::AnthropicMessages)
.await?;
Ok(())
}
@@ -1395,13 +1371,7 @@ mod tests {
});
client
- .call_rewrite_model_raw(
- Context::default(),
- raw,
- None,
- Some("claude"),
- WireFormat::AnthropicMessages,
- )
+ .call_rewrite_model_raw(raw, None, Some("claude"), WireFormat::AnthropicMessages)
.await?;
Ok(())
}
@@ -1427,7 +1397,7 @@ mod tests {
let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
let response = client
- .call_rewrite_model(Context::default(), request_for(Some("gpt"), true), None)
+ .call_rewrite_model(request_for(Some("gpt"), true), None)
.await?;
assert!(matches!(response.llm_response, LlmResponse::Stream(_)));
let agg = response.llm_response.into_agg().await?;
@@ -1463,13 +1433,7 @@ mod tests {
});
let response = client
- .call_rewrite_model_raw(
- Context::default(),
- raw,
- None,
- Some("gpt"),
- WireFormat::OpenAiChat,
- )
+ .call_rewrite_model_raw(raw, None, Some("gpt"), WireFormat::OpenAiChat)
.await?;
assert!(matches!(response, RawResponse::Stream(_)));
Ok(())
@@ -1487,7 +1451,7 @@ mod tests {
let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
let Err(error) = client
- .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .call_rewrite_model(request_for(Some("gpt"), false), None)
.await
else {
panic!("expected an error");
@@ -1521,7 +1485,7 @@ mod tests {
let client =
TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 1))?;
let response = client
- .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .call_rewrite_model(request_for(Some("gpt"), false), None)
.await?;
let agg = response.llm_response.into_agg().await?;
@@ -1547,7 +1511,7 @@ mod tests {
let client =
TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 2))?;
let Err(error) = client
- .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .call_rewrite_model(request_for(Some("gpt"), false), None)
.await
else {
panic!("expected an upstream error");
@@ -1583,7 +1547,7 @@ mod tests {
let client =
TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 2))?;
let Err(error) = client
- .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .call_rewrite_model(request_for(Some("gpt"), false), None)
.await
else {
panic!("expected retry exhaustion");
@@ -1623,7 +1587,7 @@ mod tests {
.timeout(Duration::from_millis(100))
.build()?;
let response = client
- .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .call_rewrite_model(request_for(Some("gpt"), false), None)
.await?;
assert_eq!(
@@ -1721,12 +1685,9 @@ mod tests {
client.client = reqwest::Client::builder()
.timeout(std::time::Duration::from_millis(10))
.build()?;
- let decision = std::sync::Arc::new(fixed_decision("gpt"));
+ let decision = fixed_decision("gpt");
- let Err(error) = client
- .call(Context::default(), request_for(None, false), decision)
- .await
- else {
+ let Err(error) = client.call(request_for(None, false), decision).await else {
panic!("expected a timeout");
};
let LlmClientError::Timeout { source } = error else {
@@ -1753,7 +1714,7 @@ mod tests {
let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
let Err(error) = client
- .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .call_rewrite_model(request_for(Some("gpt"), false), None)
.await
else {
panic!("expected an error");
@@ -1813,9 +1774,7 @@ mod tests {
// Matchers assert forwarded x-request-id survives and reserved
// authorization is the backend's, not the client's.
- client
- .call_rewrite_model(Context::default(), request, None)
- .await?;
+ client.call_rewrite_model(request, None).await?;
let received = server
.received_requests()
.await
@@ -1851,11 +1810,9 @@ mod tests {
.await;
let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
- let decision = std::sync::Arc::new(fixed_decision("gpt"));
+ let decision = fixed_decision("gpt");
// Called through the trait; the request has no model, so "gpt" comes from the decision.
- let response = client
- .call(Context::default(), request_for(None, false), decision)
- .await?;
+ let response = client.call(request_for(None, false), decision).await?;
let agg = response.llm_response.into_agg().await?;
assert_eq!(completion_text(&agg), "routed hi");
Ok(())
@@ -1866,13 +1823,7 @@ mod tests {
-> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
let client = TranslatingLlmClient::new(&[])?;
let Err(error) = client
- .call_rewrite_model_raw(
- Context::default(),
- json!("invalid"),
- None,
- Some("gpt"),
- WireFormat::OpenAiChat,
- )
+ .call_rewrite_model_raw(json!("invalid"), None, Some("gpt"), WireFormat::OpenAiChat)
.await
else {
panic!("expected request translation to fail");
@@ -1912,13 +1863,7 @@ mod tests {
"messages": [{"role": "user", "content": "hi"}]
});
let RawResponse::Buffered(body) = client
- .call_rewrite_model_raw(
- Context::default(),
- raw,
- None,
- Some("gpt"),
- WireFormat::OpenAiChat,
- )
+ .call_rewrite_model_raw(raw, None, Some("gpt"), WireFormat::OpenAiChat)
.await?
else {
panic!("expected a buffered response");
@@ -1954,13 +1899,7 @@ mod tests {
"stream": true
});
let RawResponse::Stream(stream) = client
- .call_rewrite_model_raw(
- Context::default(),
- raw,
- None,
- Some("gpt"),
- WireFormat::OpenAiChat,
- )
+ .call_rewrite_model_raw(raw, None, Some("gpt"), WireFormat::OpenAiChat)
.await?
else {
panic!("expected a streamed response");
@@ -2008,13 +1947,7 @@ mod tests {
// Matchers assert the forwarded x-request-id survives and reserved
// authorization is the backend's, not the client's.
client
- .call_rewrite_model_raw(
- Context::default(),
- raw,
- Some(headers),
- Some("gpt"),
- WireFormat::OpenAiChat,
- )
+ .call_rewrite_model_raw(raw, Some(headers), Some("gpt"), WireFormat::OpenAiChat)
.await?;
Ok(())
}
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index 092a7b04a..4427fdf3a 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -16,8 +16,8 @@ use std::sync::Arc;
use std::time::{Duration, Instant};
use parking_lot::Mutex;
-use switchyard_libsy::{Algorithm, CallModelRequest, LibsyError, Result, algorithm_label, drive};
-use switchyard_protocol::{Context, Decision, LlmClientError, Request, Response, RoutedLlmClient};
+use switchyard_libsy::{Algorithm, CallModel, LibsyError, Result, drive};
+use switchyard_protocol::{Decision, LlmClientError, Request, Response, RoutedLlmClient};
use crate::observation::{LlmCallObservation, RunObservation, RunObserver};
use crate::{metrics, observability};
@@ -38,16 +38,15 @@ use crate::{metrics, observability};
pub async fn run(
algorithm: Arc<dyn Algorithm>,
clients: ClientRouter,
- ctx: Context,
request: Request,
observer: Option<RunObserver>,
-) -> Result<(Vec<Arc<Decision>>, Response)> {
+) -> Result<(Vec<Decision>, Response)> {
let algorithm_name = algorithm.name().to_string();
// The output from `serve` goes in here: when each successful routed call was in
// flight. Everything else the run spent time on is routing overhead.
let routed_calls = Arc::new(Mutex::new(RoutedCallWindows::default()));
let run_started = Instant::now();
- let result = drive(algorithm, ctx, request, {
+ let result = drive(algorithm, request, {
let observer = observer.clone();
let routed_calls = Arc::clone(&routed_calls);
move |call| {
@@ -108,14 +107,14 @@ impl RoutedCallWindows {
name = "libsy.client_call",
skip_all,
fields(
- algorithm = algorithm_label(&call.get_routed().ctx),
- switchyard.algorithm = algorithm_label(&call.get_routed().ctx),
- selected_model = call.get_decision().selected_model_id(),
+ algorithm = call.algorithm,
+ switchyard.algorithm = call.algorithm,
+ selected_model = call.decision.selected_model_id(),
otel.kind = "client",
- otel.name = %format_args!("chat {}", call.get_decision().selected_model_id()),
+ otel.name = %format_args!("chat {}", call.decision.selected_model_id()),
openinference.span.kind = "LLM",
gen_ai.operation.name = "chat",
- gen_ai.request.model = call.get_decision().selected_model_id(),
+ gen_ai.request.model = call.decision.selected_model_id(),
gen_ai.request.stream = tracing::field::Empty,
gen_ai.request.temperature = tracing::field::Empty,
gen_ai.request.top_p = tracing::field::Empty,
@@ -141,15 +140,14 @@ impl RoutedCallWindows {
)]
async fn serve(
clients: ClientRouter,
- call: CallModelRequest,
+ call: CallModel,
observer: Option<RunObserver>,
// Output parameter because `drive` takes a function that returns a plain `Result<()>`.
routed_calls: Arc<Mutex<RoutedCallWindows>>,
) -> Result<()> {
let span = tracing::Span::current();
- observability::record_gen_ai_request(&span, &call.get_routed().request.llm_request);
+ observability::record_gen_ai_request(&span, &call.request.llm_request);
if let Some(session_id) = call
- .get_routed()
.request
.metadata
.as_ref()
@@ -157,19 +155,16 @@ async fn serve(
{
span.record("gen_ai.conversation.id", session_id);
}
- let routed = call.get_routed().clone();
- let target = routed.decision.selected_model_id().to_string();
- let is_answer_call = call.get_decision().is_answer_call();
+ let request = call.request.clone();
+ let decision = call.decision.clone();
+ let target = decision.selected_model_id().to_string();
+ let is_answer_call = decision.is_answer_call();
// Resolved before the clock starts: picking the client is Switchyard's work, not
// the provider's, so it belongs in the routing overhead.
let client = clients.route(&target);
let started = Instant::now();
let result = match client {
- Ok(client) => {
- client
- .call(routed.ctx, routed.request, routed.decision)
- .await
- }
+ Ok(client) => client.call(request, decision).await,
Err(error) => Err(error),
}
.map_err(|source| LibsyError::client_call(target, source));
@@ -179,7 +174,7 @@ async fn serve(
let result = observability::observe_client_call(result);
if let Some(observer) = observer {
observer(RunObservation::LlmCall(LlmCallObservation {
- selected_model: call.get_decision().selected_model_id().to_string(),
+ selected_model: call.decision.selected_model_id().to_string(),
is_answer_call,
is_success: result.is_ok(),
duration,
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs
index 340c0a240..a457ba996 100644
--- a/crates/libsy-llm-client/tests/observability.rs
+++ b/crates/libsy-llm-client/tests/observability.rs
@@ -34,12 +34,12 @@ use tracing_subscriber::registry::LookupSpan;
use switchyard_libsy::{
Algorithm, Driver, LibsyError, LlmClassifierConfig, LlmTarget, LlmTargetSet, LlmTaskClassifier,
- PickerMode, RoutedRequest, StageRouter, StageRouterConfig, Step, TaskClassifierConfig,
+ PickerMode, StageRouter, StageRouterConfig, Step, TaskClassifierConfig,
};
use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver};
use switchyard_protocol::{
- ContentBlock, Context, Decision, LlmRequest, LlmResponse, Message, Metadata, Request, Response,
- Role, RoutedLlmClient, ToolCall, ToolResult, Usage, WireFormat,
+ ContentBlock, Decision, LlmRequest, LlmResponse, Message, Metadata, Request, Response, Role,
+ RoutedLlmClient, ToolCall, ToolResult, Usage, WireFormat,
};
use switchyard_protocol::{
LlmClientError, LlmResponseChunk, LlmResponseStreamEvent, StopReason, text_request,
@@ -334,9 +334,8 @@ struct ClassifierClient {
impl RoutedLlmClient for ClassifierClient {
async fn call(
&self,
- _ctx: Context,
_request: Request,
- decision: Arc<Decision>,
+ decision: Decision,
) -> Result<Response, LlmClientError> {
let model = decision.selected_model_id().to_string();
let completion = if decision.is_answer_call() {
@@ -368,9 +367,8 @@ struct JudgeClient {
impl RoutedLlmClient for JudgeClient {
async fn call(
&self,
- _ctx: Context,
_request: Request,
- decision: Arc<Decision>,
+ decision: Decision,
) -> Result<Response, LlmClientError> {
if decision.is_answer_call() {
return Ok(Response {
@@ -409,9 +407,8 @@ impl RoutedLlmClient for JudgeClient {
impl RoutedLlmClient for UsageClient {
async fn call(
&self,
- _ctx: Context,
_request: Request,
- decision: Arc<Decision>,
+ decision: Decision,
) -> Result<Response, switchyard_protocol::LlmClientError> {
let mut response = text_response(
Some(decision.selected_model_id().to_string()),
@@ -442,7 +439,6 @@ impl Algorithm for SingleCallAlgo {
async fn create_run_task(
self: Arc<Self>,
- ctx: Context,
driver: Driver,
request: Request,
) -> switchyard_libsy::Result<Response> {
@@ -452,19 +448,13 @@ impl Algorithm for SingleCallAlgo {
.first()
.ok_or(LibsyError::NoTargets)?
.clone();
- let decision = Arc::new(Decision::new(
+ let decision = Decision::new(
target.semantic_name.clone(),
Some(format!("picked '{}'", target.semantic_name)),
true,
- ));
- driver.info(ctx.clone(), decision.clone()).await?;
- driver
- .call_model(RoutedRequest {
- request,
- decision,
- ctx,
- })
- .await
+ );
+ driver.info(decision.clone()).await?;
+ driver.call_model(request, decision).await
}
}
@@ -499,15 +489,8 @@ async fn run(
algorithm: Arc<dyn Algorithm>,
client: Arc<dyn RoutedLlmClient>,
request: Request,
-) -> switchyard_libsy::Result<(Vec<Arc<Decision>>, Response)> {
- switchyard_llm_client::run(
- algorithm,
- ClientRouter::single(client),
- Context::default(),
- request,
- None,
- )
- .await
+) -> switchyard_libsy::Result<(Vec<Decision>, Response)> {
+ switchyard_llm_client::run(algorithm, ClientRouter::single(client), request, None).await
}
fn classifier_router(
@@ -918,7 +901,6 @@ async fn observed_run_reports_one_successful_routed_call() -> switchyard_libsy::
let (_, response) = switchyard_llm_client::run(
algo(ALGO, MODEL),
ClientRouter::single(client),
- Context::default(),
request_with_metadata("observed-session", "observed-correlation"),
Some(observer),
)
@@ -954,9 +936,8 @@ struct StreamingUsageClient;
impl RoutedLlmClient for StreamingUsageClient {
async fn call(
&self,
- _ctx: Context,
_request: Request,
- decision: Arc<Decision>,
+ decision: Decision,
) -> Result<Response, LlmClientError> {
let usage = Usage {
input_tokens: Some(13),
@@ -987,9 +968,8 @@ struct TimeoutClient;
impl RoutedLlmClient for TimeoutClient {
async fn call(
&self,
- _ctx: Context,
_request: Request,
- _decision: Arc<Decision>,
+ _decision: Decision,
) -> Result<Response, LlmClientError> {
Err(LlmClientError::Timeout {
source: Box::new(TestError("upstream timed out")),
@@ -1107,10 +1087,7 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::
u64_gauge_value(&before, "switchyard.total_errors").unwrap_or_default();
// The call is offloaded and we fail it by hand, without a client.
- let stream = algo(ALGO, MODEL).run_stream(
- Context::default(),
- request_with_metadata("obs-session-2", "obs-corr-2"),
- );
+ let stream = algo(ALGO, MODEL).run_stream(request_with_metadata("obs-session-2", "obs-corr-2"));
tokio::pin!(stream);
let mut saw_error_step = false;
@@ -1246,7 +1223,7 @@ async fn classifier_metrics_count_only_the_final_routed_call() -> switchyard_lib
&snapshots,
"switchyard.llm_calls",
&[
- ("algorithm", ""),
+ ("algorithm", "llm_task_classifier"),
("selected_model", "classifier"),
("outcome", "ok"),
],
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index 4c44c571c..2f172560a 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -17,6 +17,7 @@
//! remembered for the rest of its session and skipped on later turns.
//! An unavailable target is skipped only for the current request.
+use std::collections::HashSet;
use std::{
collections::HashMap,
sync::{Arc, Once, Weak},
@@ -33,7 +34,7 @@ use crate::core::algorithm::{
use crate::core::classifier::{Classification, Classifier, Score};
use crate::core::processor::{Event, Processor};
use crate::{LibsyError, Result};
-use switchyard_protocol::{Context, Decision, Request, Response, RoutingFallbackReason};
+use switchyard_protocol::{Decision, Request, Response, RoutingFallbackReason};
struct SessionState<S> {
state: Arc<AsyncMutex<S>>,
@@ -161,12 +162,7 @@ where
self
}
/// Executes the processor/classifier/target-call sequence for wrappers and the trait entrypoint.
- pub(crate) async fn execute(
- &self,
- ctx: Context,
- driver: Driver,
- request: Request,
- ) -> Result<Response> {
+ pub(crate) async fn execute(&self, driver: Driver, request: Request) -> Result<Response> {
self.start_cleanup_task();
let session = session_id(&request);
let session_final = request
@@ -174,7 +170,7 @@ where
.as_ref()
.and_then(|metadata| metadata.session_final)
== Some(true);
- let result = self.execute_session(ctx, driver, request).await;
+ let result = self.execute_session(driver, request).await;
if session_final && let Some(session) = session.as_deref() {
self.remove_session(session);
}
@@ -192,20 +188,17 @@ where
});
}
- async fn execute_session(
- &self,
- ctx: Context,
- driver: Driver,
- request: Request,
- ) -> Result<Response> {
+ async fn execute_session(&self, driver: Driver, request: Request) -> Result<Response> {
// The request is threaded mutably through the whole fold: any component may rewrite
// it, later components see the rewrite, and the final value reaches the model.
let mut request = request;
- let mut ctx = ctx;
+ // Targets this turn must not route to. Scratch state for one run: seeded from the
+ // session's overflow history, then grown by each route-level failure below.
+ let mut excluded = HashSet::new();
// Processors may rewrite the request; overflow history stays with its inbound identity.
let identity = RoutingIdentity::from_request(&request);
algorithm::exclude_evicted(
- &mut ctx,
+ &mut excluded,
&self.targets,
&self.session_evictions,
identity.as_ref(),
@@ -214,11 +207,13 @@ where
let (target, decision, served, deciding) = match session_state {
Some(state) => {
let mut state = state.lock().await;
- self.route(&mut state, &ctx, &driver, &mut request).await?
+ self.route(&mut state, &excluded, &driver, &mut request)
+ .await?
}
None => {
let mut state = S::default();
- self.route(&mut state, &ctx, &driver, &mut request).await?
+ self.route(&mut state, &excluded, &driver, &mut request)
+ .await?
}
};
@@ -231,7 +226,7 @@ where
Some(response) => Ok(response),
None => {
algorithm::call_model_with_fallback(
- ctx,
+ &mut excluded,
&driver,
&self.targets,
target,
@@ -266,12 +261,12 @@ where
from: &LlmTarget,
to: &LlmTarget,
reason: RoutingFallbackReason,
- ) -> Arc<Decision> {
+ ) -> Decision {
let failure = match reason {
RoutingFallbackReason::ContextWindow => "exceeded its context window",
RoutingFallbackReason::Unavailable => "was unavailable",
};
- Arc::new(Decision::new(
+ Decision::new(
to.semantic_name.clone(),
Some(with_routing_tier(
format!(
@@ -283,7 +278,7 @@ where
deciding.routing_tier(&to.semantic_name),
)),
true,
- ))
+ )
}
/// Returns this request's retained state without holding the registry lock.
@@ -303,12 +298,12 @@ where
async fn route(
&self,
state: &mut S,
- ctx: &Context,
+ excluded: &HashSet<String>,
driver: &Driver,
request: &mut Request,
) -> Result<(
LlmTarget,
- Arc<Decision>,
+ Decision,
Option<Response>,
Arc<dyn Classifier<S>>,
)> {
@@ -337,7 +332,7 @@ where
// 3. Resolve the target and publish the decision. When an excluded target sends
// the request elsewhere, the reasoning describes where it actually went.
- let target = self.targets.resolve_target(&score.target, ctx)?;
+ let target = self.targets.resolve_target(&score.target, excluded)?;
let reasoning = if target.semantic_name == score.target {
(self.decision_reason)(&self.name, &score)
} else {
@@ -346,22 +341,22 @@ where
score.target, target.semantic_name
)
};
- let decision: Arc<Decision> = Arc::new(Decision::new(
+ let decision: Decision = Decision::new(
target.semantic_name.clone(),
Some(with_routing_tier(
reasoning,
deciding.routing_tier(&target.semantic_name),
)),
true,
- ));
- driver.info(ctx.clone(), decision.clone()).await?;
+ );
+ driver.info(decision.clone()).await?;
// 4. Post-decision replay: every processor sees the decision so stateful ones
// can bind it, and may rewrite the outbound request (e.g. add a target prompt).
for processor in &self.processors {
let event = Event::Decision {
request,
- decision: decision.as_ref(),
+ decision: &decision,
};
processor.process(state, event).await?;
}
@@ -431,11 +426,10 @@ where
async fn create_run_task(
self: Arc<Self>,
- ctx: Context,
driver: Driver,
request: Request,
) -> Result<Response> {
- self.execute(ctx, driver, request).await
+ self.execute(driver, request).await
}
}
#[cfg(test)]
@@ -463,7 +457,7 @@ mod tests {
/// Echoes the routed model name, capturing the request it was handed so a test can
/// assert on what actually reached the model.
fn capturing(into: Arc<Mutex<Option<Request>>>) -> impl Serve {
- move |decision: Arc<Decision>, request: Request| {
+ move |decision: Decision, request: Request| {
let into = Arc::clone(&into);
async move {
*into.lock() = Some(request);
@@ -491,7 +485,7 @@ mod tests {
impl PromptRecorder {
fn serve(self: &Arc<Self>) -> impl Serve {
let recorder = Arc::clone(self);
- move |decision: Arc<Decision>, request: Request| {
+ move |decision: Decision, request: Request| {
let recorder = Arc::clone(&recorder);
async move {
*recorder.0.lock() = Some(RecordedCall {
@@ -546,7 +540,6 @@ mod tests {
) -> Result<RecordedCall> {
test_drive(
Arc::new(router),
- Context::default(),
Request {
llm_request: text_request(Some("auto".to_string()), "fix the build"),
raw_request: None,
@@ -626,12 +619,11 @@ mod tests {
router: &Arc<FallThrough<S>>,
request: Request,
serve: impl Serve,
- ) -> Result<(String, Vec<Arc<Decision>>)>
+ ) -> Result<(String, Vec<Decision>)>
where
S: Default + Send + 'static,
{
- let (trace, response) =
- test_drive(router.clone(), Context::default(), request, serve).await?;
+ let (trace, response) = test_drive(router.clone(), request, serve).await?;
let text = response
.llm_response
.into_agg()
@@ -645,7 +637,7 @@ mod tests {
async fn run_turn<S>(
router: &Arc<FallThrough<S>>,
serve: impl Serve,
- ) -> Result<(String, Vec<Arc<Decision>>)>
+ ) -> Result<(String, Vec<Decision>)>
where
S: Default + Send + 'static,
{
@@ -653,10 +645,7 @@ mod tests {
}
/// Drives a fresh router through one turn with a specific `serve`.
- async fn run_with(
- router: FallThrough,
- serve: impl Serve,
- ) -> Result<(String, Vec<Arc<Decision>>)> {
+ async fn run_with(router: FallThrough, serve: impl Serve) -> Result<(String, Vec<Decision>)> {
run_turn(&Arc::new(router), serve).await
}
@@ -671,7 +660,7 @@ mod tests {
overflowing: &'static [&'static str],
calls: Arc<Mutex<Vec<String>>>,
) -> impl Serve {
- move |decision: Arc<Decision>, _request: Request| {
+ move |decision: Decision, _request: Request| {
let calls = Arc::clone(&calls);
async move {
let model = decision.selected_model_id().to_string();
@@ -693,7 +682,7 @@ mod tests {
unavailable: &'static [&'static str],
calls: Arc<Mutex<Vec<String>>>,
) -> impl Serve {
- move |decision: Arc<Decision>, _request: Request| {
+ move |decision: Decision, _request: Request| {
let calls = Arc::clone(&calls);
async move {
let model = decision.selected_model_id().to_string();
@@ -982,21 +971,16 @@ mod tests {
}
#[tokio::test]
- async fn an_excluded_target_reports_the_target_it_fell_back_to() -> Result<()> {
+ async fn a_target_barred_by_overflow_history_reports_the_fallback() -> Result<()> {
// Headers and usage metrics read the decision, so it must describe the real call.
let router = Arc::new(
FallThrough::<()>::new(target_set(&["weak", "strong"]))
.with_classifier(fixed(vec![score("weak", 0.9)])),
);
- let mut ctx = Context::default();
- ctx.exclude_target("weak");
- let (trace, response) = test_drive(router, ctx, request(), echo()).await?;
- let text = response
- .llm_response
- .into_agg()
- .await
- .map(|agg| completion_text(&agg))
- .map_err(|error| LibsyError::external("aggregating fall-through response", error))?;
+ // The first turn teaches the session that "weak" overflows; the second is barred
+ // from it before calling, so routing redirects and must say where it went.
+ run_turn(&router, overflows(&["weak"])).await?;
+ let (text, trace) = run_turn(&router, overflows(&["weak"])).await?;
assert_eq!(text, "strong");
assert_eq!(trace[0].selected_model_id(), "strong");
@@ -1284,7 +1268,7 @@ mod tests {
..request()
};
- let result = test_drive(router.clone(), Context::default(), final_request, echo()).await;
+ let result = test_drive(router.clone(), final_request, echo()).await;
assert!(matches!(result, Err(LibsyError::AlgorithmError { .. })));
let states = router
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index 725d91c88..be0536df7 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -21,13 +21,11 @@ use super::util::llm_judge::{
SerdeDecoder, StructuredJudge,
};
use super::util::target_selector::TargetSelectorPolicy;
-use crate::core::algorithm::{Algorithm, Driver, LlmTarget, LlmTargetSet, RoutedRequest};
+use crate::core::algorithm::{Algorithm, Driver, LlmTarget, LlmTargetSet};
use crate::core::classifier::{Classification, Classifier, Score};
use crate::core::state::{State, StateValue};
use crate::{LibsyError, Result};
-use switchyard_protocol::{
- AggLlmResponse, Context, LlmClientError, LlmResponse, Request, Response,
-};
+use switchyard_protocol::{AggLlmResponse, LlmClientError, LlmResponse, Request, Response};
const PROMPT_TEMPLATE: &str = include_str!("../prompts/capability-classifier/prompt.md");
const SCHEMA_TEMPLATE: &str = include_str!("../prompts/capability-classifier/schema.json");
@@ -511,23 +509,19 @@ impl Classifier<State> for EscalationClassifier {
}
// Call efficient model and buffer the response so the judge can read it.
- // `Classifier::score` takes no `ctx`, so inner calls use Context::default() and their
- // spans carry algorithm="" rather than the algorithm name. Known gap shared with the
- // task classifier's judge consultation.
//
// If the efficient model exceeds its context window, fall through to capable: returning
// `(decisive(capable), None)` tells FallThrough::execute to call
// call_model_with_fallback with the capable target instead of surfacing the error.
let efficient_response = match driver
- .call_model(RoutedRequest {
- request: request.clone(),
- decision: Arc::new(Decision::new(
+ .call_model(
+ request.clone(),
+ Decision::new(
self.efficient.semantic_name.clone(),
Some("escalation classifier: efficient tier".into()),
true,
- )),
- ctx: Context::default(),
- })
+ ),
+ )
.await
{
Ok(r) => r,
@@ -938,11 +932,10 @@ impl Algorithm for LlmTaskClassifier {
async fn create_run_task(
self: Arc<Self>,
- ctx: Context,
driver: Driver,
request: Request,
) -> Result<Response> {
- self.route.execute(ctx, driver, request).await
+ self.route.execute(driver, request).await
}
}
@@ -961,7 +954,7 @@ mod tests {
use crate::algorithms::util::llm_judge::Judge;
use crate::core::testing::{Serve, reply, test_drive};
- use switchyard_protocol::{Context, LlmResponse, Response};
+ use switchyard_protocol::{LlmResponse, Response};
const TEST_THRESHOLD: f64 = 0.5;
@@ -1031,7 +1024,7 @@ mod tests {
fn serve(self: &Arc<Self>) -> impl Serve {
let recorder = Arc::clone(self);
- move |decision: Arc<Decision>, request: Request| {
+ move |decision: Decision, request: Request| {
let recorder = Arc::clone(&recorder);
async move {
let model = decision.selected_model_id().to_string();
@@ -1075,7 +1068,7 @@ mod tests {
/// The judge times out; every other target answers normally.
fn unreachable_judge() -> impl Serve {
- |decision: Arc<Decision>, request: Request| async move {
+ |decision: Decision, request: Request| async move {
let model = decision.selected_model_id().to_string();
if model == "judge" {
return Err(LlmClientError::Timeout {
@@ -1138,13 +1131,7 @@ mod tests {
async fn an_unreachable_judge_routes_capable_instead_of_failing_the_request() -> Result<()> {
let router = router()?;
- let (trace, response) = test_drive(
- router,
- Context::default(),
- classify_request(),
- unreachable_judge(),
- )
- .await?;
+ let (trace, response) = test_drive(router, classify_request(), unreachable_judge()).await?;
assert_eq!(trace.last().map(|d| d.selected_model_id()), Some("capable"));
assert_eq!(
@@ -1160,20 +1147,8 @@ mod tests {
let router = router()?;
let request = classify_request;
- test_drive(
- router.clone(),
- Context::default(),
- request(),
- recorder.serve(),
- )
- .await?;
- test_drive(
- router.clone(),
- Context::default(),
- request(),
- recorder.serve(),
- )
- .await?;
+ test_drive(router.clone(), request(), recorder.serve()).await?;
+ test_drive(router.clone(), request(), recorder.serve()).await?;
assert_eq!(
recorder.calls(),
@@ -1207,13 +1182,7 @@ mod tests {
},
})?);
- test_drive(
- router,
- Context::default(),
- classify_request(),
- recorder.serve(),
- )
- .await?;
+ test_drive(router, classify_request(), recorder.serve()).await?;
assert_eq!(recorder.judge_max_output_tokens(), vec![Some(512)]);
Ok(())
@@ -1236,13 +1205,7 @@ mod tests {
},
})?);
- test_drive(
- router,
- Context::default(),
- classify_request(),
- recorder.serve(),
- )
- .await?;
+ test_drive(router, classify_request(), recorder.serve()).await?;
let prompts = recorder.judge_system_prompts();
assert_eq!(prompts.len(), 1);
@@ -1267,20 +1230,8 @@ mod tests {
})?);
let session_request = classify_session_request;
- test_drive(
- router.clone(),
- Context::default(),
- session_request(),
- recorder.serve(),
- )
- .await?;
- test_drive(
- router.clone(),
- Context::default(),
- session_request(),
- recorder.serve(),
- )
- .await?;
+ test_drive(router.clone(), session_request(), recorder.serve()).await?;
+ test_drive(router.clone(), session_request(), recorder.serve()).await?;
assert_eq!(recorder.calls(), vec!["judge", "efficient", "efficient"]);
Ok(())
@@ -1304,16 +1255,9 @@ mod tests {
},
})?);
+ test_drive(router.clone(), classify_request(), recorder.serve()).await?;
test_drive(
router.clone(),
- Context::default(),
- classify_request(),
- recorder.serve(),
- )
- .await?;
- test_drive(
- router.clone(),
- Context::default(),
classify_follow_up_request(),
recorder.serve(),
)
@@ -1837,7 +1781,7 @@ mod tests {
/// Serves the judge target from `judge` and every other target from `model`, each with
/// its next queued reply.
fn queued(model: Arc<Queue>, judge: Arc<Queue>) -> impl Serve {
- move |decision: Arc<Decision>, request: Request| {
+ move |decision: Decision, request: Request| {
let queue = if decision.selected_model_id() == "judge" {
Arc::clone(&judge)
} else {
@@ -1879,13 +1823,8 @@ mod tests {
let model = Queue::new(["efficient answer"]);
let router = escalation_router()?;
- let (trace, response) = test_drive(
- router,
- Context::default(),
- classify_request(),
- queued(model, judge),
- )
- .await?;
+ let (trace, response) =
+ test_drive(router, classify_request(), queued(model, judge)).await?;
// The efficient model is the serving target, and the response comes from its call.
assert_eq!(
@@ -1923,13 +1862,7 @@ mod tests {
max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,
})?);
- test_drive(
- router,
- Context::default(),
- classify_request(),
- recorder.serve(),
- )
- .await?;
+ test_drive(router, classify_request(), recorder.serve()).await?;
let prompts = recorder.judge_system_prompts();
assert_eq!(prompts.len(), 1);
@@ -1945,13 +1878,8 @@ mod tests {
let model = Queue::new(["efficient draft", "capable answer"]);
let router = escalation_router()?;
- let (trace, response) = test_drive(
- router,
- Context::default(),
- classify_request(),
- queued(model, judge),
- )
- .await?;
+ let (trace, response) =
+ test_drive(router, classify_request(), queued(model, judge)).await?;
assert_eq!(trace.last().map(|d| d.selected_model_id()), Some("capable"));
assert!(
@@ -1978,18 +1906,11 @@ mod tests {
let session_request = classify_session_request();
test_drive(
router.clone(),
- Context::default(),
session_request.clone(),
queued(Arc::clone(&model), Arc::clone(&judge)),
)
.await?;
- let (trace, _) = test_drive(
- router.clone(),
- Context::default(),
- session_request,
- queued(model, judge),
- )
- .await?;
+ let (trace, _) = test_drive(router.clone(), session_request, queued(model, judge)).await?;
assert_eq!(trace.last().map(|d| d.selected_model_id()), Some("capable"));
Ok(())
@@ -2003,7 +1924,7 @@ mod tests {
let router = escalation_router()?;
// Efficient overflows, capable answers, and the judge must never be called.
- let serve = |decision: Arc<Decision>, _request: Request| async move {
+ let serve = |decision: Decision, _request: Request| async move {
match decision.selected_model_id() {
"efficient" => Err(LlmClientError::ContextWindowExceeded {
model: decision.selected_model_id().to_string(),
@@ -2014,8 +1935,7 @@ mod tests {
}
};
- let (trace, response) =
- test_drive(router, Context::default(), classify_request(), serve).await?;
+ let (trace, response) = test_drive(router, classify_request(), serve).await?;
assert_eq!(trace.last().map(|d| d.selected_model_id()), Some("capable"));
assert_eq!(
diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rs
index 973f7e0b5..64810991b 100644
--- a/crates/libsy/src/algorithms/noop.rs
+++ b/crates/libsy/src/algorithms/noop.rs
@@ -11,7 +11,7 @@ use switchyard_protocol::{
use crate::Result;
use crate::core::algorithm::{Algorithm, Driver};
-use switchyard_protocol::{Context, Decision};
+use switchyard_protocol::Decision;
/// Test helper that returns a hard-coded response without routing or model I/O.
pub struct Noop {}
@@ -24,7 +24,6 @@ impl Algorithm for Noop {
async fn create_run_task(
self: Arc<Self>,
- ctx: Context,
driver: Driver,
request: Request,
) -> Result<Response> {
@@ -32,12 +31,12 @@ impl Algorithm for Noop {
.requested_model()
.unwrap_or("switchyard/noop")
.to_string();
- let decision: Arc<Decision> = Arc::new(Decision::new(
+ let decision: Decision = Decision::new(
model.clone(),
Some("noop returned its synthetic response".to_string()),
true,
- ));
- driver.info(ctx, decision.clone()).await?;
+ );
+ driver.info(decision.clone()).await?;
let llm_response = LlmResponse::Agg(AggLlmResponse {
id: Some("switchyard-noop".to_string()),
@@ -82,7 +81,7 @@ mod tests {
// `Noop` synthesizes its own response and never offloads a call, so `echo` is
// never reached.
let a: Arc<dyn Algorithm> = Arc::new(Noop {});
- let (decisions, response) = test_drive(a, Context::default(), request, echo()).await?;
+ let (decisions, response) = test_drive(a, request, echo()).await?;
let Some(decision) = decisions.first() else {
panic!("Expected exactly one Decision");
};
diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rs
index cab9140bd..f2d9f7f64 100644
--- a/crates/libsy/src/algorithms/passthrough.rs
+++ b/crates/libsy/src/algorithms/passthrough.rs
@@ -8,8 +8,8 @@ use std::sync::Arc;
use switchyard_protocol::{Request, Response};
use crate::Result;
-use crate::core::algorithm::{Algorithm, Driver, LlmTarget, RoutedRequest};
-use switchyard_protocol::{Context, Decision};
+use crate::core::algorithm::{Algorithm, Driver, LlmTarget};
+use switchyard_protocol::Decision;
/// Routing algorithm that always calls one configured target.
pub struct Passthrough {
@@ -31,26 +31,19 @@ impl Algorithm for Passthrough {
async fn create_run_task(
self: Arc<Self>,
- ctx: Context,
driver: Driver,
request: Request,
) -> Result<Response> {
- let decision: Arc<Decision> = Arc::new(Decision::new(
+ let decision: Decision = Decision::new(
self.target.semantic_name.clone(),
Some(format!(
"passthrough selected target '{}'",
self.target.semantic_name
)),
true,
- ));
- driver.info(ctx.clone(), decision.clone()).await?;
- driver
- .call_model(RoutedRequest {
- request,
- decision,
- ctx,
- })
- .await
+ );
+ driver.info(decision.clone()).await?;
+ driver.call_model(request, decision).await
}
}
@@ -61,7 +54,7 @@ mod tests {
use super::Passthrough;
use crate::core::algorithm::{Algorithm, LlmTarget};
use crate::core::testing::{echo, test_drive};
- use switchyard_protocol::{Context, Request, completion_text, text_request};
+ use switchyard_protocol::{Request, completion_text, text_request};
#[tokio::test]
async fn test_passthrough() -> crate::Result<()> {
@@ -74,7 +67,7 @@ mod tests {
let algorithm: Arc<dyn Algorithm> = Arc::new(Passthrough::new(LlmTarget {
semantic_name: MODEL_ID.to_string(),
}));
- let (trace, response) = test_drive(algorithm, Context::default(), request, echo()).await?;
+ let (trace, response) = test_drive(algorithm, request, echo()).await?;
assert_eq!(
response
diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rs
index d724a8de3..96f3bbed8 100644
--- a/crates/libsy/src/algorithms/rand.rs
+++ b/crates/libsy/src/algorithms/rand.rs
@@ -19,7 +19,7 @@ use crate::algorithms::fall_through::FallThrough;
use crate::core::algorithm::{Algorithm, Driver, LlmTargetSet};
use crate::core::classifier::{Classification, Classifier, Score};
use crate::{LibsyError, Result};
-use switchyard_protocol::{Context, Request, Response};
+use switchyard_protocol::{Request, Response};
/// Stateless weighted classifier used by random fall-through routing.
pub struct RandomClassifier {
@@ -161,11 +161,10 @@ impl Algorithm for Random {
async fn create_run_task(
self: Arc<Self>,
- ctx: Context,
driver: Driver,
request: Request,
) -> Result<Response> {
- self.inner.execute(ctx, driver, request).await
+ self.inner.execute(driver, request).await
}
}
@@ -220,8 +219,7 @@ mod tests {
async fn selected_models(algorithm: Arc<dyn Algorithm>, count: usize) -> Result<Vec<String>> {
let mut selected = Vec::with_capacity(count);
for _ in 0..count {
- let (_, response) =
- test_drive(algorithm.clone(), Context::default(), request(), echo()).await?;
+ let (_, response) = test_drive(algorithm.clone(), request(), echo()).await?;
selected.push(
response
.llm_response
@@ -236,8 +234,7 @@ mod tests {
#[tokio::test]
async fn single_target_is_always_selected_and_called() -> Result<()> {
let algorithm = shared_algorithm(&["only/model"])?;
- let (trace, response) =
- test_drive(algorithm, Context::default(), request(), echo()).await?;
+ let (trace, response) = test_drive(algorithm, request(), echo()).await?;
assert_eq!(
response
@@ -258,8 +255,7 @@ mod tests {
let algorithm = shared_algorithm(&names)?;
for _ in 0..50 {
- let (trace, response) =
- test_drive(algorithm.clone(), Context::default(), request(), echo()).await?;
+ let (trace, response) = test_drive(algorithm.clone(), request(), echo()).await?;
let selected = response
.llm_response
.as_agg()
@@ -280,8 +276,7 @@ mod tests {
let mut seen = HashSet::new();
for _ in 0..100 {
- let (_, response) =
- test_drive(algorithm.clone(), Context::default(), request(), echo()).await?;
+ let (_, response) = test_drive(algorithm.clone(), request(), echo()).await?;
seen.insert(
response
.llm_response
@@ -345,13 +340,8 @@ mod tests {
.with_classifier(random),
);
- let (_, first) = test_drive(
- algorithm.clone(),
- Context::default(),
- request_for_session("session-1"),
- echo(),
- )
- .await?;
+ let (_, first) =
+ test_drive(algorithm.clone(), request_for_session("session-1"), echo()).await?;
let selected = first
.llm_response
.as_agg()
@@ -370,13 +360,7 @@ mod tests {
Some(selected.to_string())
);
- let (_, second) = test_drive(
- algorithm,
- Context::default(),
- request_for_session("session-1"),
- echo(),
- )
- .await?;
+ let (_, second) = test_drive(algorithm, request_for_session("session-1"), echo()).await?;
assert_eq!(
second
.llm_response
@@ -428,7 +412,7 @@ mod tests {
#[tokio::test]
async fn decision_is_inspectable() -> Result<()> {
let algorithm = shared_algorithm(&["only/model"])?;
- let (trace, _) = test_drive(algorithm, Context::default(), request(), echo()).await?;
+ let (trace, _) = test_drive(algorithm, request(), echo()).await?;
let decision = &trace[0];
assert_eq!(decision.selected_model_id(), "only/model");
diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs
index e25f06b37..f90611e8c 100644
--- a/crates/libsy/src/algorithms/stage.rs
+++ b/crates/libsy/src/algorithms/stage.rs
@@ -29,7 +29,7 @@ use crate::core::algorithm::{Algorithm, Driver, LlmTarget, LlmTargetSet};
use crate::core::classifier::{Classification, Classifier};
use crate::core::state::State;
use crate::{LibsyError, Result};
-use switchyard_protocol::{Context, Request, Response};
+use switchyard_protocol::{Request, Response};
/// Telemetry name for a router this module assembles.
const STAGE_ROUTER: &str = "stage_router";
@@ -145,11 +145,10 @@ impl Algorithm for StageRouter {
async fn create_run_task(
self: Arc<Self>,
- ctx: Context,
driver: Driver,
request: Request,
) -> Result<Response> {
- self.route.execute(ctx, driver, request).await
+ self.route.execute(driver, request).await
}
}
@@ -232,7 +231,7 @@ mod tests {
use crate::core::classifier::Score;
use crate::core::state::StateValue;
use crate::core::testing::{Serve, reply, test_drive};
- use switchyard_protocol::{Context, Decision, Metadata, Response};
+ use switchyard_protocol::{Decision, Metadata, Response};
fn tier_target(name: &str) -> LlmTarget {
LlmTarget {
@@ -379,7 +378,7 @@ mod tests {
/// back so the fallback classifier has an answer without a real model.
fn serve(self: &Arc<Self>) -> impl Serve {
let recorder = Arc::clone(self);
- move |decision: Arc<Decision>, request: Request| {
+ move |decision: Decision, request: Request| {
let recorder = Arc::clone(&recorder);
async move {
let target = decision.selected_model_id().to_string();
@@ -494,16 +493,9 @@ mod tests {
async fn a_signal_driven_escalation_hands_the_note_to_the_model() -> Result<()> {
let recorder = Arc::new(Recorder::default());
let router = recording_router(config_with_notes())?;
- let ctx = Context::default();
- test_drive(
- router.clone(),
- ctx.clone(),
- turn_request(false),
- recorder.serve(),
- )
- .await?;
- test_drive(router.clone(), ctx, turn_request(true), recorder.serve()).await?;
+ test_drive(router.clone(), turn_request(false), recorder.serve()).await?;
+ test_drive(router.clone(), turn_request(true), recorder.serve()).await?;
let calls = recorder.routed();
assert_eq!(calls[0].target, "weak");
@@ -529,13 +521,7 @@ mod tests {
let recorder = Arc::new(Recorder::default());
let router = recording_router(config_with_judge(&recorder, 0.1))?;
- let (trace, _) = test_drive(
- router.clone(),
- Context::default(),
- turn_request(false),
- recorder.serve(),
- )
- .await?;
+ let (trace, _) = test_drive(router.clone(), turn_request(false), recorder.serve()).await?;
let calls = recorder.calls.lock();
assert!(
@@ -565,13 +551,7 @@ mod tests {
let recorder = Arc::new(Recorder::default());
let router = recording_router(config_with_judge(&recorder, 0.9))?;
- test_drive(
- router.clone(),
- Context::default(),
- turn_request(true),
- recorder.serve(),
- )
- .await?;
+ test_drive(router.clone(), turn_request(true), recorder.serve()).await?;
assert!(
!recorder.calls.lock().iter().any(|c| c.target == JUDGE),
@@ -585,17 +565,10 @@ mod tests {
async fn the_judges_verdict_is_not_pinned_to_the_session() -> Result<()> {
let recorder = Arc::new(Recorder::default());
let router = recording_router(config_with_judge(&recorder, 0.1))?;
- let ctx = Context::default();
-
- test_drive(
- router.clone(),
- ctx.clone(),
- turn_request(false),
- recorder.serve(),
- )
- .await?;
+
+ test_drive(router.clone(), turn_request(false), recorder.serve()).await?;
*recorder.judge_p_solve.lock() = 0.9;
- test_drive(router.clone(), ctx, turn_request(false), recorder.serve()).await?;
+ test_drive(router.clone(), turn_request(false), recorder.serve()).await?;
let routed = recorder.routed();
assert_eq!(routed[0].target, "strong");
@@ -618,13 +591,7 @@ mod tests {
let recorder = Arc::new(Recorder::default());
let router = recording_router(config_with_judge(&recorder, 42.0))?;
- test_drive(
- router.clone(),
- Context::default(),
- turn_request(false),
- recorder.serve(),
- )
- .await?;
+ test_drive(router.clone(), turn_request(false), recorder.serve()).await?;
assert_eq!(recorder.routed()[0].target, "weak");
Ok(())
@@ -635,13 +602,7 @@ mod tests {
let recorder = Arc::new(Recorder::default());
let router = recording_router(config_with_judge(&recorder, 0.9))?;
- test_drive(
- router.clone(),
- Context::default(),
- turn_request(false),
- recorder.serve(),
- )
- .await?;
+ test_drive(router.clone(), turn_request(false), recorder.serve()).await?;
let judged = recorder
.calls
diff --git a/crates/libsy/src/algorithms/subagent_affinity_tests.rs b/crates/libsy/src/algorithms/subagent_affinity_tests.rs
index 6fde55d14..02b808934 100644
--- a/crates/libsy/src/algorithms/subagent_affinity_tests.rs
+++ b/crates/libsy/src/algorithms/subagent_affinity_tests.rs
@@ -19,7 +19,7 @@ use crate::core::algorithm::{Driver, LlmTarget, LlmTargetSet};
use crate::core::classifier::{Classification, Classifier, Score};
use crate::core::testing::{echo, test_drive};
use switchyard_protocol::{
- Context, Metadata, Request, Response, completion_text, slice_to_header_map, text_request,
+ Metadata, Request, Response, completion_text, slice_to_header_map, text_request,
};
/// The cascade's terminal classifier: always picks the orchestrator.
@@ -77,8 +77,7 @@ fn router() -> Arc<FallThrough> {
/// Runs one turn, returning the target that served it.
async fn turn(router: &Arc<FallThrough>, headers: &[(&str, &str)]) -> Result<String> {
- let (_, response) =
- test_drive(router.clone(), Context::default(), request(headers), echo()).await?;
+ let (_, response) = test_drive(router.clone(), request(headers), echo()).await?;
Ok(response
.llm_response
.as_agg()
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
index ae47123d7..464544e24 100644
--- a/crates/libsy/src/algorithms/util/llm_judge.rs
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -8,7 +8,6 @@
//! the route.
use std::marker::PhantomData;
-use std::sync::Arc;
use async_trait::async_trait;
use serde::de::DeserializeOwned;
@@ -18,11 +17,11 @@ use switchyard_protocol::{
};
use super::classifier_contract::ClassifierContract;
-use crate::core::algorithm::{Driver, LlmTarget, RoutedRequest};
+use crate::core::algorithm::{Driver, LlmTarget};
use crate::core::classifier::{Classification, Classifier};
use crate::core::state::State;
use crate::{LibsyError, Result};
-use switchyard_protocol::{Context, Decision, LlmClientError, Request, Response};
+use switchyard_protocol::{Decision, LlmClientError, Request, Response};
/// Builds the classifier-specific message view presented to a structured judge.
pub(crate) trait ClassifierInput: Send + Sync {
@@ -227,15 +226,14 @@ where
let judge_model = self.target.semantic_name.as_str();
let response = driver
- .call_model(RoutedRequest {
- request: self.judge.build_request(state, request),
- decision: Arc::new(Decision::new(
+ .call_model(
+ self.judge.build_request(state, request),
+ Decision::new(
self.target.semantic_name.to_string(),
Some("llm judge consultation".to_string()),
false,
- )),
- ctx: Context::default(),
- })
+ ),
+ )
.await
.inspect_err(|error| report_fail_open(judge_model, error, libsy_error_reason(error)))
.ok()?;
@@ -463,7 +461,7 @@ mod tests {
/// Serves the single offloaded judge call with `reply` through a standalone step receiver.
async fn score_served_with(reply: Result<Response>) -> Result<String> {
- let (driver, step_rx) = Driver::new();
+ let (driver, step_rx) = Driver::new("test");
let mut steps = tokio_stream::wrappers::ReceiverStream::new(step_rx);
let classifier = classifier();
let mut state = State::default();
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index 43820d5e1..34a33fee1 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -28,7 +28,7 @@ use tracing::Instrument;
/// [`switchyard_protocol::LlmResponse`] carries either a live
/// [`switchyard_protocol::LlmResponseStream`] or the terminal aggregate.
use switchyard_protocol::{
- Context, Decision, LlmClientError, Request, Response, RoutingFallbackReason, Signals,
+ Decision, LlmClientError, Request, Response, RoutingFallbackReason, Signals,
};
use crate::{DriverError, LibsyError, Result, observability};
@@ -38,54 +38,31 @@ use crate::{DriverError, LibsyError, Result, observability};
/// `Arc<dyn Algorithm>` object-safe.
pub type StepStream = Pin<Box<dyn Stream<Item = Result<Step>> + Send>>;
-/// A request paired with the routing [`Decision`] that produced it — the offload
-/// payload a host reads (via [`CallModelRequest::get_routed`]) to serve the call.
+/// An offloaded model call, surfaced inside [`Step::CallModel`].
+///
+/// The host reads the public fields, performs (or delegates) the model call, and fulfills it
+/// with [`respond`](Self::respond) — unblocking the algorithm's [`Driver::call_model`] on the
+/// other side. `switchyard-llm-client`'s `run` is the ready-made consumer that does this for
+/// you. A host that only wants the routing outcome can take the contents with
+/// [`into_parts`](Self::into_parts) and never respond; dropping the stream ends the run.
///
/// The selected model and inbound route name live in separate, unambiguous places: the model
/// identifier is [`decision.selected_model_id()`](Decision::selected_model_id), while
/// `request.llm_request.model` is the *inbound* name the agent asked for (libsy
/// never overwrites it). A client maps `selected_model_id()` to the provider model
/// id it hits.
-#[derive(Clone)]
-pub struct RoutedRequest {
+pub struct CallModel {
+ /// The name of the algorithm that produced this call, so a host instrumenting the
+ /// calls it serves can attribute its own spans to the algorithm behind them.
+ pub algorithm: String,
/// The request to serve; its `model` is the agent's original name.
pub request: Request,
/// The routing decision behind this call; `selected_model_id()` identifies the model to hit.
- pub decision: Arc<Decision>,
- /// The request's cross-cutting context, carried through the offload so the host
- /// serving the call can pass it to its client.
- pub ctx: Context,
-}
-
-/// The host-facing half of an offloaded model call, surfaced inside [`Step::CallModel`].
-///
-/// The host reads the routed request ([`get_routed`](Self::get_routed)) and the decision
-/// behind it ([`get_decision`](Self::get_decision)), performs (or delegates) the model
-/// call, and fulfills it with [`respond`](Self::respond) — unblocking the algorithm's
-/// [`Driver::call_model`] on the other side. `switchyard-llm-client`'s `run` is the ready-made
-/// consumer that does this for you.
-pub struct CallModelRequest {
- routed: RoutedRequest,
+ pub decision: Decision,
reply: oneshot::Sender<Result<Response>>,
}
-impl CallModelRequest {
- /// The routed request the host should serve; its `decision.selected_model_id()` names
- /// the model to hit.
- pub fn get_routed(&self) -> &RoutedRequest {
- &self.routed
- }
-
- /// The model request to perform (the [`Request`] inside the routed request).
- pub fn get_request(&self) -> &Request {
- &self.get_routed().request
- }
-
- /// The decision that led to this call; its `selected_model_id()` identifies the model to hit.
- pub fn get_decision(&self) -> &Decision {
- self.get_routed().decision.as_ref()
- }
-
+impl CallModel {
/// Fulfill the promise with the caller's model-call result. Pass `Err(..)` to
/// propagate a failed model call back to the algorithm. Consumes the promise: it
/// can only be fulfilled once.
@@ -94,6 +71,30 @@ impl CallModelRequest {
.send(result)
.map_err(|_| DriverError::ResponseDropped.into())
}
+
+ /// Take the call's contents without answering it, dropping the promise — the routing
+ /// outcome plus the request as the algorithm would have sent it, after any rewriting.
+ ///
+ /// Should only be called if `decision.is_answer_call` is true as that is the final call.
+ /// The algorithm's [`Driver::call_model`] will fail with [`DriverError::Abandoned`] and
+ /// the run ends there. Taking a call the algorithm does not depend on (a judge or
+ /// classifier call) may instead let it fail open and complete with degraded routing.
+ ///
+ /// An abandoned run is not recorded as a failed one. Dropping a [`CallModel`] without
+ /// calling this still yields [`DriverError::ResponseDropped`], which is.
+ pub fn into_parts(self) -> (Request, Decision) {
+ let Self {
+ request,
+ decision,
+ reply,
+ ..
+ } = self;
+ // Tell the algorithm the call was taken deliberately rather than lost, so its
+ // telemetry can tell an abandoned run from a failed one. The receiver is already
+ // gone if the algorithm stopped waiting, which is fine.
+ let _ = reply.send(Err(DriverError::Abandoned.into()));
+ (request, decision)
+ }
}
/// The offload channel handed to an algorithm's
@@ -105,23 +106,31 @@ impl CallModelRequest {
#[derive(Clone)]
pub struct Driver {
step_tx: mpsc::Sender<Result<Step>>,
+ /// The owning algorithm's telemetry label, stamped onto every call and decision
+ /// this driver publishes.
+ algorithm: String,
}
impl Driver {
/// Build an empty driver with its step channel ready. Created per call by
/// [`run_stream`](Algorithm::run_stream). Also returns the Step receiver.
- pub(crate) fn new() -> (Self, mpsc::Receiver<Result<Step>>) {
+ pub(crate) fn new(algorithm: &str) -> (Self, mpsc::Receiver<Result<Step>>) {
// Capacity one keeps the algorithm paced by the stream consumer. It limits queued steps,
// not model calls already pulled from the stream, which can still run at the same time.
// A larger buffer would use more memory and let the algorithm run farther ahead with
// little benefit because reading a step is cheap compared with serving a model call.
let (step_tx, step_rx) = mpsc::channel(1);
- (Self { step_tx }, step_rx)
+ (
+ Self {
+ step_tx,
+ algorithm: algorithm.to_string(),
+ },
+ step_rx,
+ )
}
- /// Offload a model call: publish `routed` as a [`Step::CallModel`] and await the
- /// consumer's [`Response`]. The call's context travels inside
- /// [`routed.ctx`](RoutedRequest::ctx). Errors if the stream is closed or the call failed.
+ /// Offload a model call: publish it as a [`Step::CallModel`] and await the consumer's
+ /// [`Response`]. Errors if the stream is closed or the call failed.
/// The await is wrapped in a `libsy.llm_call` span measuring *fulfillment* as
/// the algorithm observes it (host queueing/serving included; a streamed
/// response resolves when its stream handle arrives); latency, outcome, and
@@ -132,8 +141,8 @@ impl Driver {
name = "libsy.llm_call",
skip_all,
fields(
- algorithm = observability::algorithm_label(&routed.ctx),
- selected_model = routed.decision.selected_model_id(),
+ algorithm = self.algorithm,
+ selected_model = decision.selected_model_id(),
openinference.span.kind = "CHAIN",
outcome = tracing::field::Empty,
error = tracing::field::Empty,
@@ -143,13 +152,17 @@ impl Driver {
reasoning_tokens = tracing::field::Empty,
)
)]
- pub async fn call_model(&self, routed: RoutedRequest) -> Result<Response> {
- let algorithm = observability::algorithm_label(&routed.ctx).to_string();
- let decision = Arc::clone(&routed.decision);
+ pub async fn call_model(&self, request: Request, decision: Decision) -> Result<Response> {
+ let selected_model_id = decision.selected_model_id().to_string();
let is_answer_call = decision.is_answer_call();
let started = Instant::now();
let (reply, response) = oneshot::channel::<Result<Response>>();
- let call = CallModelRequest { routed, reply };
+ let call = CallModel {
+ algorithm: self.algorithm.clone(),
+ request,
+ decision,
+ reply,
+ };
let result = async {
self.step_tx
.send(Ok(Step::CallModel(Box::new(call))))
@@ -162,8 +175,8 @@ impl Driver {
.await;
let elapsed = started.elapsed();
observability::record_llm_call(
- &algorithm,
- decision.selected_model_id(),
+ &self.algorithm,
+ &selected_model_id,
is_answer_call,
elapsed,
&result,
@@ -175,12 +188,12 @@ impl Driver {
/// Publish a routing [`Decision`] as a [`Step::Decision`] on the stream.
/// Each successfully published decision is counted and logged with its
/// reasoning; a decision the stream never accepted is not recorded.
- pub async fn info(&self, ctx: Context, decision: Arc<Decision>) -> Result<()> {
+ pub async fn info(&self, decision: Decision) -> Result<()> {
self.step_tx
.send(Ok(Step::Decision(decision.clone())))
.await
.map_err(|_| DriverError::StreamClosed)?;
- observability::record_decision(&ctx, decision.as_ref());
+ observability::record_decision(&self.algorithm, &decision);
Ok(())
}
@@ -199,11 +212,11 @@ impl Driver {
/// One item in the stream returned by [`Algorithm::run_stream`].
pub enum Step {
/// The algorithm needs this model call performed. The host serves it and fulfills
- /// it with [`CallModelRequest::respond`]. Boxed: it is by far the largest variant.
- CallModel(Box<CallModelRequest>),
+ /// it with [`CallModel::respond`]. Boxed: it is by far the largest variant.
+ CallModel(Box<CallModel>),
/// A routing decision the algorithm made, published via [`Driver::info`] as it
/// happens (rather than collected into a trace returned at the end).
- Decision(Arc<Decision>),
+ Decision(Decision),
/// The algorithm finished with its final response — the last step of a run.
ReturnToAgent(Box<Response>),
}
@@ -212,7 +225,7 @@ pub enum Step {
///
/// Returns the final [`Response`] and the trace of [`Decision`]s the algorithm published.
/// `serve` owns the call: it performs it however the host likes and must fulfill the promise
-/// with [`CallModelRequest::respond`]. A failed *model* call belongs in `respond` — the
+/// with [`CallModel::respond`]. A failed *model* call belongs in `respond` — the
/// algorithm may route around it. Returning `Err` from `serve` aborts the whole run, so
/// reserve it for infrastructure failures. Calls are served concurrently, so an algorithm
/// that offloads several at once (hedging, fan-out) gets real parallelism.
@@ -222,18 +235,17 @@ pub enum Step {
/// is this function plus an HTTP client.
pub async fn drive<F, Fut>(
algorithm: Arc<dyn Algorithm>,
- ctx: Context,
request: Request,
serve: F,
-) -> Result<(Vec<Arc<Decision>>, Response)>
+) -> Result<(Vec<Decision>, Response)>
where
- F: Fn(CallModelRequest) -> Fut,
+ F: Fn(CallModel) -> Fut,
Fut: Future<Output = Result<()>>,
{
- let stream = algorithm.run_stream(ctx, request);
+ let stream = algorithm.run_stream(request);
tokio::pin!(stream);
- let mut trace: Vec<Arc<Decision>> = Vec::new();
+ let mut trace: Vec<Decision> = Vec::new();
let mut in_flight = futures::stream::FuturesUnordered::new();
let mut final_response: Option<Response> = None;
@@ -274,7 +286,7 @@ impl Drop for AbortOnDrop {
/// A named routing target an algorithm routes by. Serving its calls is the stream
/// consumer's concern: the selected identifier reaches the consumer as
-/// `decision.selected_model_id()` on the offloaded [`RoutedRequest`].
+/// `decision.selected_model_id()` on the offloaded [`CallModel`].
#[derive(Clone)]
pub struct LlmTarget {
/// The routing label an algorithm selects this target by — a logical tier like
@@ -313,16 +325,16 @@ impl LlmTargetSet {
})
}
- /// The named target, or the first one this request is not barred from when it has been
- /// excluded (see [`Context::exclude_target`]). Errors if every target is excluded.
- pub fn resolve_target(&self, name: &str, ctx: &Context) -> Result<LlmTarget> {
+ /// The named target, or the first one not in `excluded` when it has been barred.
+ /// Errors if every target is excluded.
+ pub fn resolve_target(&self, name: &str, excluded: &HashSet<String>) -> Result<LlmTarget> {
let target = self.get_target(name)?;
- if !ctx.is_excluded(&target.semantic_name) {
+ if !excluded.contains(&target.semantic_name) {
return Ok(target);
}
self.targets
.iter()
- .find(|t| !ctx.is_excluded(&t.semantic_name))
+ .find(|t| !excluded.contains(&t.semantic_name))
.cloned()
.ok_or(LibsyError::AllTargetsExcluded)
}
@@ -419,18 +431,18 @@ impl SessionEvictions {
}
/// How many of `targets` this request is still allowed to reach.
-fn eligible_targets(targets: &LlmTargetSet, ctx: &Context) -> usize {
+fn eligible_targets(targets: &LlmTargetSet, excluded: &HashSet<String>) -> usize {
targets
.targets()
.iter()
- .filter(|t| !ctx.is_excluded(&t.semantic_name))
+ .filter(|t| !excluded.contains(&t.semantic_name))
.count()
}
/// Bars the targets `identity` has already overflowed from this request, so routing does
/// not select one that is certain to fail again.
pub(crate) fn exclude_evicted(
- ctx: &mut Context,
+ excluded: &mut HashSet<String>,
targets: &LlmTargetSet,
evictions: &SessionEvictions,
identity: Option<&RoutingIdentity>,
@@ -438,10 +450,10 @@ pub(crate) fn exclude_evicted(
for target in evictions.evicted_for(identity) {
// Never seed the pool empty: a later turn may be small enough to serve, and the
// caller should get the upstream's answer rather than a routing error.
- if eligible_targets(targets, ctx) <= 1 {
+ if eligible_targets(targets, excluded) <= 1 {
break;
}
- ctx.exclude_target(target);
+ excluded.insert(target);
}
}
@@ -474,44 +486,38 @@ fn classify_fallback(error: &LibsyError) -> Option<(&str, RoutingFallbackReason)
/// overflows are recorded for `identity`; unavailable targets remain request-local.
#[allow(clippy::too_many_arguments)]
pub(crate) async fn call_model_with_fallback(
- mut ctx: Context,
+ excluded: &mut HashSet<String>,
driver: &Driver,
targets: &LlmTargetSet,
mut target: LlmTarget,
- mut decision: Arc<Decision>,
+ mut decision: Decision,
request: Request,
identity: Option<&RoutingIdentity>,
evictions: &SessionEvictions,
target_unavailable: impl Fn(&Request, &str),
- fallback_decision: impl Fn(&LlmTarget, &LlmTarget, RoutingFallbackReason) -> Arc<Decision>,
+ fallback_decision: impl Fn(&LlmTarget, &LlmTarget, RoutingFallbackReason) -> Decision,
) -> Result<Response> {
loop {
- let result = driver
- .call_model(RoutedRequest {
- request: request.clone(),
- decision: decision.clone(),
- ctx: ctx.clone(),
- })
- .await;
+ let result = driver.call_model(request.clone(), decision.clone()).await;
let Err(error) = result else { return result };
let Some((failed, reason)) = classify_fallback(&error) else {
return Err(error);
};
// A target already excluded means the pool is spent; surface the client error
// so the caller still sees the concrete upstream failure.
- if !ctx.exclude_target(failed) {
+ if !excluded.insert(failed.to_string()) {
return Err(error);
}
match reason {
RoutingFallbackReason::ContextWindow => evictions.record(identity, failed),
RoutingFallbackReason::Unavailable => target_unavailable(&request, failed),
}
- let Ok(next) = targets.resolve_target(&target.semantic_name, &ctx) else {
+ let Ok(next) = targets.resolve_target(&target.semantic_name, excluded) else {
return Err(error);
};
decision = fallback_decision(&target, &next, reason);
target = next;
- driver.info(ctx.clone(), decision.clone()).await?;
+ driver.info(decision.clone()).await?;
}
}
@@ -546,14 +552,8 @@ pub trait Algorithm: Send + Sync + 'static {
/// Run one request to completion: make model calls with [`Driver::call_model`],
/// publish [`Decision`]s with [`Driver::info`], and return the final [`Response`].
/// The method an algorithm implements; [`run_stream`](Self::run_stream) drives it.
- /// `ctx` carries the request's cross-cutting values (today: the algorithm's
- /// telemetry label in [`Context::values`]).
- async fn create_run_task(
- self: Arc<Self>,
- ctx: Context,
- driver: Driver,
- request: Request,
- ) -> Result<Response>;
+ async fn create_run_task(self: Arc<Self>, driver: Driver, request: Request)
+ -> Result<Response>;
/// Feed the algorithm agentic-stack events (tool results, budgets, etc.). The
/// reference algorithms ignore signals; a stateful algorithm updates its own
@@ -571,17 +571,9 @@ pub trait Algorithm: Send + Sync + 'static {
/// emitted as an `Err` item. Dropping the stream aborts the spawned algorithm task.
///
/// Every invocation owns a separate [`Driver`].
- fn run_stream(self: Arc<Self>, ctx: Context, request: Request) -> StepStream {
- // Stamp the algorithm's telemetry label into the request context; the
- // context rides on every driver call, so its telemetry is attributed.
- let mut ctx = ctx;
- ctx.values.insert(
- observability::ALGORITHM_KEY.to_string(),
- self.name().to_string(),
- );
- let (driver, step_rx) = Driver::new();
+ fn run_stream(self: Arc<Self>, request: Request) -> StepStream {
+ let (driver, step_rx) = Driver::new(self.name());
let task_driver = driver.clone();
- let task_ctx = ctx.clone();
let stream = ReceiverStream::new(step_rx);
// One `libsy.run` span covers the whole algorithm task; the driver's
// `libsy.llm_call` spans and decision logs nest inside it via `tracing`'s
@@ -589,11 +581,9 @@ pub trait Algorithm: Send + Sync + 'static {
let span = observability::run_span(self.name(), &request);
let handle = tokio::spawn(
async move {
- observability::observe_run(
- task_ctx.clone(),
- self.create_run_task(task_ctx, task_driver, request),
- )
- .await
+ let algorithm = self.name().to_string();
+ observability::observe_run(&algorithm, self.create_run_task(task_driver, request))
+ .await
}
.instrument(span),
);
@@ -694,8 +684,8 @@ mod tests {
}
/// Build a routed decision for orchestration tests.
- fn test_decision(selected_model_id: String) -> Arc<Decision> {
- Arc::new(Decision::new(selected_model_id, None, true))
+ fn test_decision(selected_model_id: String) -> Decision {
+ Decision::new(selected_model_id, None, true)
}
/// Trivial algo used only to exercise the orchestrator: calls the first target
@@ -712,7 +702,6 @@ mod tests {
async fn create_run_task(
self: Arc<Self>,
- ctx: Context,
driver: Driver,
request: Request,
) -> Result<Response> {
@@ -723,14 +712,8 @@ mod tests {
.ok_or(LibsyError::NoTargets)?
.clone();
let decision = test_decision(target.semantic_name.clone());
- driver.info(ctx.clone(), decision.clone()).await?;
- driver
- .call_model(RoutedRequest {
- request,
- decision,
- ctx,
- })
- .await
+ driver.info(decision.clone()).await?;
+ driver.call_model(request, decision).await
}
}
@@ -757,24 +740,23 @@ mod tests {
LlmTargetSet::new(targets)
}
- fn routed(model: &str) -> RoutedRequest {
- RoutedRequest {
- request: request(),
- decision: test_decision(model.to_string()),
- ctx: Context::default(),
- }
- }
-
#[tokio::test]
async fn typed_driver_preserves_call_and_stream_boundaries() -> Result<()> {
tokio::time::timeout(std::time::Duration::from_secs(1), async {
// Distinct oneshots keep reverse-order replies paired with their producers, and a
// retained call remains pending until the host responds.
- let (driver, mut step_rx) = Driver::new();
+ let (driver, mut step_rx) = Driver::new("test");
let first_driver = driver.clone();
- let mut first =
- tokio::spawn(async move { first_driver.call_model(routed("first")).await });
- let second = tokio::spawn(async move { driver.call_model(routed("second")).await });
+ let mut first = tokio::spawn(async move {
+ first_driver
+ .call_model(request(), test_decision("first".to_string()))
+ .await
+ });
+ let second = tokio::spawn(async move {
+ driver
+ .call_model(request(), test_decision("second".to_string()))
+ .await
+ });
let mut calls = HashMap::new();
for _ in 0..2 {
@@ -782,7 +764,7 @@ mod tests {
let Step::CallModel(call) = step else {
return Err(test_error("expected a CallModel step"));
};
- calls.insert(call.get_decision().selected_model_id().to_string(), call);
+ calls.insert(call.decision.selected_model_id().to_string(), call);
}
assert!(
tokio::time::timeout(std::time::Duration::from_millis(20), &mut first)
@@ -815,8 +797,12 @@ mod tests {
);
// Dropping the host-facing promise closes only that call's reply channel.
- let (driver, mut step_rx) = Driver::new();
- let producer = tokio::spawn(async move { driver.call_model(routed("dropped")).await });
+ let (driver, mut step_rx) = Driver::new("test");
+ let producer = tokio::spawn(async move {
+ driver
+ .call_model(request(), test_decision("dropped".to_string()))
+ .await
+ });
let step = step_rx.recv().await.ok_or(DriverError::StreamClosed)??;
let Step::CallModel(call) = step else {
return Err(test_error("expected a CallModel step"));
@@ -831,10 +817,10 @@ mod tests {
));
// A standalone driver reports the typed step receiver disappearing at its next send.
- let (driver, step_rx) = Driver::new();
+ let (driver, step_rx) = Driver::new("test");
drop(step_rx);
let decision = test_decision("closed".to_string());
- let result = driver.info(Context::default(), decision).await;
+ let result = driver.info(decision).await;
assert!(matches!(
result,
Err(LibsyError::Driver(DriverError::StreamClosed))
@@ -845,6 +831,50 @@ mod tests {
.map_err(|error| LibsyError::external("waiting for typed driver boundaries", error))?
}
+ /// A consumer that only wants the routing outcome takes the call apart instead of
+ /// answering it: it gets the request as the algorithm would have sent it, and the
+ /// algorithm learns the call was abandoned rather than lost — the distinction that
+ /// keeps a deliberate decision-only run out of the failure counters.
+ #[tokio::test]
+ async fn into_parts_yields_the_call_without_answering_it() -> Result<()> {
+ let (driver, mut step_rx) = Driver::new("test");
+ let decision = test_decision("answer/model".to_string());
+ let producer = tokio::spawn({
+ let decision = decision.clone();
+ async move { driver.call_model(request(), decision).await }
+ });
+
+ let step = step_rx.recv().await.ok_or(DriverError::StreamClosed)??;
+ let Step::CallModel(call) = step else {
+ return Err(test_error("expected a CallModel step"));
+ };
+ let (taken_request, taken_decision) = call.into_parts();
+ assert_eq!(taken_decision.selected_model_id(), "answer/model");
+ assert!(taken_decision.is_answer_call());
+ assert_eq!(taken_request.llm_request, request().llm_request);
+
+ let result = producer
+ .await
+ .map_err(|source| LibsyError::AlgorithmTask { source })?;
+ assert!(matches!(
+ result,
+ Err(LibsyError::Driver(DriverError::Abandoned))
+ ));
+ Ok(())
+ }
+
+ /// The two ways a call goes unanswered are told apart: `into_parts` is the consumer's
+ /// choice, a bare drop is the promise being lost.
+ #[test]
+ fn abandoning_and_dropping_a_call_report_different_outcomes() {
+ let abandoned: Result<Response> = Err(DriverError::Abandoned.into());
+ let dropped: Result<Response> = Err(DriverError::ResponseDropped.into());
+ assert_eq!(observability::outcome_value(&abandoned), "abandoned");
+ assert_eq!(observability::outcome_value(&dropped), "error");
+ assert!(observability::is_abandoned(&abandoned));
+ assert!(!observability::is_abandoned(&dropped));
+ }
+
#[test]
fn target_lookup_returns_the_missing_target() {
let error = target_set(&[]).get_target("missing").err();
@@ -858,7 +888,7 @@ mod tests {
/// replaying `chunks` in order (as `Ok` items).
fn streaming_orch(chunks: Vec<LlmResponseChunk>) -> (Arc<dyn Algorithm>, impl Serve) {
let algo = orch(target_set(&["stream/model"]));
- let serve = move |_decision: Arc<Decision>, _request: Request| {
+ let serve = move |_decision: Decision, _request: Request| {
let chunks = chunks.clone();
async move {
let stream =
@@ -893,7 +923,7 @@ mod tests {
reason: Some("stop".to_string()),
},
]);
- let (trace, response) = test_drive(orch, Context::default(), request(), serve).await?;
+ let (trace, response) = test_drive(orch, request(), serve).await?;
// The run handed back the live stream; the caller folds it to a buffered aggregate.
let agg = response
.llm_response
@@ -919,7 +949,7 @@ mod tests {
message: "upstream exploded".to_string(),
},
]);
- let (_, response) = test_drive(orch, Context::default(), request(), serve).await?;
+ let (_, response) = test_drive(orch, request(), serve).await?;
match response.llm_response.into_agg().await {
Ok(_) => panic!("expected a mid-stream error, got an aggregate"),
Err(err) => {
@@ -933,7 +963,7 @@ mod tests {
async fn run_offloads_via_promise_then_returns_to_agent() -> Result<()> {
// Every call is offloaded via a promise the orchestrator surfaces as a
// `CallModel` step for us to fulfill.
- let stream = orch(target_set(&["offload/model"])).run_stream(Context::default(), request());
+ let stream = orch(target_set(&["offload/model"])).run_stream(request());
tokio::pin!(stream);
let mut saw_call = false;
@@ -943,7 +973,7 @@ mod tests {
Step::CallModel(call) => {
saw_call = true;
// The decision rode along with the promise.
- assert_eq!(call.get_decision().selected_model_id(), "offload/model");
+ assert_eq!(call.decision.selected_model_id(), "offload/model");
// Fulfilling the promise is the "real" model call the caller makes.
call.respond(Ok(Response {
llm_response: LlmResponse::Agg(text_response(
@@ -978,13 +1008,8 @@ mod tests {
#[tokio::test]
async fn a_driven_run_returns_the_trace_and_the_final_response() -> Result<()> {
- let (trace, response) = test_drive(
- orch(target_set(&["direct/model"])),
- Context::default(),
- request(),
- echo(),
- )
- .await?;
+ let (trace, response) =
+ test_drive(orch(target_set(&["direct/model"])), request(), echo()).await?;
// TestAlgo calls the first target; `echo` answers with its name.
assert_eq!(
response
@@ -1017,7 +1042,7 @@ mod tests {
for _ in 0..N {
let algo = algo.clone();
let barrier = barrier.clone();
- let serve = move |decision: Arc<Decision>, _request: Request| {
+ let serve = move |decision: Decision, _request: Request| {
let barrier = barrier.clone();
async move {
barrier.wait().await;
@@ -1025,7 +1050,7 @@ mod tests {
}
};
handles.push(tokio::spawn(async move {
- test_drive(algo, Context::default(), request(), serve)
+ test_drive(algo, request(), serve)
.await
.map(|(_, response)| {
response
@@ -1053,7 +1078,7 @@ mod tests {
// A client-less target offloads its call; we fulfill the promise with an
// Err, which must flow back through `call_model_target` into the algorithm and
// out as an error step — not a response.
- let stream = orch(target_set(&["offload/model"])).run_stream(Context::default(), request());
+ let stream = orch(target_set(&["offload/model"])).run_stream(request());
tokio::pin!(stream);
let mut saw_error = false;
@@ -1108,7 +1133,6 @@ mod tests {
async fn create_run_task(
self: Arc<Self>,
- _ctx: Context,
_driver: Driver,
_request: Request,
) -> Result<Response> {
@@ -1127,7 +1151,7 @@ mod tests {
dropped: dropped.clone(),
});
- let stream = algo.run_stream(Context::default(), request());
+ let stream = algo.run_stream(request());
started_rx
.recv()
.await
@@ -1156,7 +1180,6 @@ mod tests {
async fn create_run_task(
self: Arc<Self>,
- _ctx: Context,
_driver: Driver,
_request: Request,
) -> Result<Response> {
@@ -1165,7 +1188,7 @@ mod tests {
}
let algo: Arc<dyn Algorithm> = Arc::new(Panicky);
- let stream = algo.run_stream(Context::default(), request());
+ let stream = algo.run_stream(request());
tokio::pin!(stream);
let mut saw_error = false;
@@ -1197,7 +1220,6 @@ mod tests {
async fn create_run_task(
self: Arc<Self>,
- _ctx: Context,
_driver: Driver,
_request: Request,
) -> Result<Response> {
@@ -1206,7 +1228,7 @@ mod tests {
}
let algo: Arc<dyn Algorithm> = Arc::new(Panicky);
- match test_drive(algo, Context::default(), request(), echo()).await {
+ match test_drive(algo, request(), echo()).await {
Ok(_) => Err(test_error(
"expected the run to surface the algorithm panic as an error",
)),
@@ -1245,7 +1267,6 @@ mod tests {
async fn create_run_task(
self: Arc<Self>,
- _ctx: Context,
_driver: Driver,
_request: Request,
) -> Result<Response> {
@@ -1267,10 +1288,7 @@ mod tests {
// Drive the run on its own task, wait until the algorithm task is up, then cancel
// it — dropping its future (and the `run_stream` stream it holds).
- let run_task =
- tokio::spawn(
- async move { test_drive(algo, Context::default(), request(), echo()).await },
- );
+ let run_task = tokio::spawn(async move { test_drive(algo, request(), echo()).await });
started_rx
.recv()
.await
@@ -1302,22 +1320,13 @@ mod tests {
async fn create_run_task(
self: Arc<Self>,
- ctx: Context,
driver: Driver,
request: Request,
) -> Result<Response> {
let dec_w = test_decision(self.winner.semantic_name.clone());
let dec_l = test_decision(self.loser.semantic_name.clone());
- let win = driver.call_model(RoutedRequest {
- request: request.clone(),
- decision: dec_w,
- ctx: ctx.clone(),
- });
- let lose = driver.call_model(RoutedRequest {
- request,
- decision: dec_l,
- ctx,
- });
+ let win = driver.call_model(request.clone(), dec_w);
+ let lose = driver.call_model(request, dec_l);
// First to resolve wins; `select!` drops the losing future (and its promise).
tokio::select! {
res = win => res,
@@ -1339,7 +1348,7 @@ mod tests {
semantic_name: "loser".to_string(),
},
});
- let serve = move |decision: Arc<Decision>, _request: Request| {
+ let serve = move |decision: Decision, _request: Request| {
let started = started.clone();
async move {
if decision.selected_model_id() == "loser" {
@@ -1362,7 +1371,7 @@ mod tests {
// The loser responds 50ms after the winner has already won. `run` must return the
// winner, not the loser's `respond`-to-a-dropped-receiver error.
let (algo, serve) = hedge(Some(std::time::Duration::from_millis(50)));
- let (_trace, response) = test_drive(algo, Context::default(), request(), serve).await?;
+ let (_trace, response) = test_drive(algo, request(), serve).await?;
assert_eq!(
response
.llm_response
@@ -1379,7 +1388,7 @@ mod tests {
// The loser never resolves. `run` must return the winner promptly, not hang
// waiting for the in-flight loser.
let (algo, serve) = hedge(None);
- let run = test_drive(algo, Context::default(), request(), serve);
+ let run = test_drive(algo, request(), serve);
let (_trace, response) = tokio::time::timeout(std::time::Duration::from_secs(1), run)
.await
.map_err(|error| LibsyError::external("waiting for pending loser", error))??;
@@ -1417,17 +1426,12 @@ mod tests {
async fn create_run_task(
self: Arc<Self>,
- ctx: Context,
driver: Driver,
request: Request,
) -> Result<Response> {
let offloads = futures::future::join_all((0..self.n).map(|i| {
let decision = test_decision(format!("m{i}"));
- driver.call_model(RoutedRequest {
- request: request.clone(),
- decision,
- ctx: ctx.clone(),
- })
+ driver.call_model(request.clone(), decision)
}));
tokio::select! {
_ = offloads => Err(test_error("offloads unexpectedly completed")),
@@ -1446,7 +1450,7 @@ mod tests {
// Serving enters each call; once all N are in flight it signals, then pends forever.
let started = Arc::new(AtomicUsize::new(0));
- let serve = move |_decision: Arc<Decision>, _request: Request| {
+ let serve = move |_decision: Decision, _request: Request| {
let started = started.clone();
let all_started = all_started.clone();
async move {
@@ -1459,7 +1463,7 @@ mod tests {
// With the cap gone, the driver keeps polling the stream even with N calls in
// flight, so the terminal error surfaces promptly instead of hanging.
- let run = test_drive(algo, Context::default(), request(), serve);
+ let run = test_drive(algo, request(), serve);
let result = tokio::time::timeout(std::time::Duration::from_millis(500), run)
.await
.map_err(|error| {
diff --git a/crates/libsy/src/core/testing.rs b/crates/libsy/src/core/testing.rs
index 2643d4b94..1f03b74cc 100644
--- a/crates/libsy/src/core/testing.rs
+++ b/crates/libsy/src/core/testing.rs
@@ -17,10 +17,10 @@ use std::sync::Arc;
use futures::future::BoxFuture;
use switchyard_protocol::{
- Context, Decision, LlmClientError, LlmResponse, Request, Response, text_response,
+ Decision, LlmClientError, LlmResponse, Request, Response, text_response,
};
-use crate::core::algorithm::{Algorithm, CallModelRequest};
+use crate::core::algorithm::{Algorithm, CallModel};
use crate::{LibsyError, Result};
/// The result a fake client hands back for one offloaded call.
@@ -29,15 +29,15 @@ pub(crate) type ServeResult = std::result::Result<Response, LlmClientError>;
/// Answers offloaded model calls. Returning `Err` propagates a failed *model* call back into
/// the algorithm, which may route around it.
pub(crate) trait Serve: Send + Sync + 'static {
- fn serve(&self, decision: Arc<Decision>, request: Request) -> BoxFuture<'static, ServeResult>;
+ fn serve(&self, decision: Decision, request: Request) -> BoxFuture<'static, ServeResult>;
}
impl<F, Fut> Serve for F
where
- F: Fn(Arc<Decision>, Request) -> Fut + Send + Sync + 'static,
+ F: Fn(Decision, Request) -> Fut + Send + Sync + 'static,
Fut: Future<Output = ServeResult> + Send + 'static,
{
- fn serve(&self, decision: Arc<Decision>, request: Request) -> BoxFuture<'static, ServeResult> {
+ fn serve(&self, decision: Decision, request: Request) -> BoxFuture<'static, ServeResult> {
Box::pin(self(decision, request))
}
}
@@ -45,12 +45,11 @@ where
/// Run `algorithm` to completion, serving each offloaded call with `serve`.
pub(crate) async fn test_drive(
algorithm: Arc<dyn Algorithm>,
- ctx: Context,
request: Request,
serve: impl Serve,
-) -> Result<(Vec<Arc<Decision>>, Response)> {
+) -> Result<(Vec<Decision>, Response)> {
let serve = Arc::new(serve);
- crate::drive(algorithm, ctx, request, move |call| {
+ crate::drive(algorithm, request, move |call| {
fulfill(Arc::clone(&serve), call)
})
.await
@@ -58,11 +57,12 @@ pub(crate) async fn test_drive(
/// Serve one call and fulfill its promise, mapping failures the way a host does so
/// error-shape assertions match production.
-async fn fulfill(serve: Arc<impl Serve>, call: CallModelRequest) -> Result<()> {
- let routed = call.get_routed().clone();
- let target = routed.decision.selected_model_id().to_string();
+async fn fulfill(serve: Arc<impl Serve>, call: CallModel) -> Result<()> {
+ let request = call.request.clone();
+ let decision = call.decision.clone();
+ let target = decision.selected_model_id().to_string();
let result = serve
- .serve(routed.decision, routed.request)
+ .serve(decision, request)
.await
.map_err(|source| LibsyError::client_call(target, source));
call.respond(result)
@@ -71,7 +71,7 @@ async fn fulfill(serve: Arc<impl Serve>, call: CallModelRequest) -> Result<()> {
/// Answers with the selected model name as the completion — what most routing tests need,
/// since they assert on *which* target was called.
pub(crate) fn echo() -> impl Serve {
- |decision: Arc<Decision>, _request: Request| async move { Ok(reply(decision.selected_model_id())) }
+ |decision: Decision, _request: Request| async move { Ok(reply(decision.selected_model_id())) }
}
/// A buffered response whose completion text is `completion`.
diff --git a/crates/libsy/src/error.rs b/crates/libsy/src/error.rs
index 1e21e950f..e77dfe80c 100644
--- a/crates/libsy/src/error.rs
+++ b/crates/libsy/src/error.rs
@@ -104,6 +104,12 @@ pub enum DriverError {
/// One side of a response promise was dropped before delivery.
#[error("driver response promise was dropped")]
ResponseDropped,
+
+ /// The consumer took the call's contents with `CallModel::into_parts` instead of
+ /// answering it, so the run ends here by the consumer's choice. Distinct from
+ /// [`ResponseDropped`](Self::ResponseDropped), which means the promise was lost.
+ #[error("model call was abandoned by the consumer")]
+ Abandoned,
}
#[cfg(test)]
diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs
index 1f2bf462f..3a48885aa 100644
--- a/crates/libsy/src/lib.rs
+++ b/crates/libsy/src/lib.rs
@@ -6,8 +6,7 @@
mod core;
pub use core::algorithm::{
- Algorithm, CallModelRequest, Driver, LlmTarget, LlmTargetSet, RoutedRequest, Step, StepStream,
- drive,
+ Algorithm, CallModel, Driver, LlmTarget, LlmTargetSet, Step, StepStream, drive,
};
pub use core::classifier::{Classification, Classifier, Score};
pub use core::processor::{Event, Processor};
@@ -41,7 +40,6 @@ pub use algorithms::util::stage::{
};
mod observability;
-pub use observability::algorithm_label;
/// Registers process-wide compatibility gauges with the global meter provider.
///
diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs
index c86e5c43c..56e55ed32 100644
--- a/crates/libsy/src/observability.rs
+++ b/crates/libsy/src/observability.rs
@@ -41,8 +41,8 @@ use opentelemetry::metrics::{Meter, ObservableGauge};
use opentelemetry::{KeyValue, global};
use tracing::Span;
-use crate::Result;
-use switchyard_protocol::{Context, Decision, Request, Response};
+use crate::{DriverError, LibsyError, Result};
+use switchyard_protocol::{Decision, Request, Response};
const METRICS_SCOPE: &str = "switchyard";
const TRACING_TARGET: &str = "libsy";
@@ -51,22 +51,6 @@ static TOTAL_REQUESTS: AtomicU64 = AtomicU64::new(0);
static TOTAL_ERRORS: AtomicU64 = AtomicU64::new(0);
static TOTAL_GAUGES: OnceLock<(ObservableGauge<u64>, ObservableGauge<u64>)> = OnceLock::new();
-/// [`Context::values`] key under which `run_stream` stamps the algorithm's
-/// telemetry label ([`Algorithm::name`](crate::Algorithm::name)).
-pub(crate) const ALGORITHM_KEY: &str = "algorithm";
-
-/// The algorithm label [`Algorithm::run_stream`](crate::Algorithm::run_stream) stamps into
-/// a request context; empty until stamped.
-///
-/// A host instrumenting the calls it serves reads this to attribute its own spans to the
-/// algorithm that produced them.
-pub fn algorithm_label(ctx: &Context) -> &str {
- ctx.values
- .get(ALGORITHM_KEY)
- .map(String::as_str)
- .unwrap_or("")
-}
-
/// The `libsy`-scoped meter from the globally installed provider.
pub(crate) fn meter() -> Meter {
global::meter(METRICS_SCOPE)
@@ -92,9 +76,18 @@ pub(crate) fn initialize_metrics() {
});
}
-/// `outcome` attribute value for a result: `ok` or `error`.
-fn outcome_value<T>(result: &Result<T>) -> &'static str {
- if result.is_ok() { "ok" } else { "error" }
+/// Whether a result is a call the consumer deliberately abandoned rather than a failure.
+pub(crate) fn is_abandoned<T>(result: &Result<T>) -> bool {
+ matches!(result, Err(LibsyError::Driver(DriverError::Abandoned)))
+}
+
+/// `outcome` attribute value for a result: `ok`, `abandoned`, or `error`.
+pub(crate) fn outcome_value<T>(result: &Result<T>) -> &'static str {
+ match result {
+ Ok(_) => "ok",
+ Err(LibsyError::Driver(DriverError::Abandoned)) => "abandoned",
+ Err(_) => "error",
+ }
}
/// Span covering one algorithm run (the whole `create_run_task` execution).
@@ -149,13 +142,12 @@ pub(crate) fn run_span(algorithm: &str, request: &Request) -> Span {
/// histogram, span outcome, and failure log when it resolves.
/// Executes inside the `libsy.run` span its caller instruments the task with.
pub(crate) async fn observe_run(
- ctx: Context,
+ algorithm: &str,
run: impl Future<Output = Result<Response>>,
) -> Result<Response> {
let started = Instant::now();
let result = run.await;
let duration = started.elapsed();
- let algorithm = algorithm_label(&ctx);
record_run(algorithm, duration, &result, &Span::current());
result
}
@@ -166,7 +158,11 @@ pub(crate) async fn observe_run(
fn record_run(algorithm: &str, duration: Duration, result: &Result<Response>, span: &Span) {
let outcome = outcome_value(result);
span.record("outcome", outcome);
- if let Err(error) = result {
+ // An abandoned run ended because the consumer took the call, not because anything went
+ // wrong, so it is counted but not reported as a failure.
+ if let Err(error) = result
+ && !is_abandoned(result)
+ {
span.record("error", tracing::field::display(error));
tracing::warn!(
target: TRACING_TARGET,
@@ -234,7 +230,9 @@ pub(crate) fn record_llm_call(
.build()
.record(duration.as_secs_f64() * 1000.0, &call_attributes);
- if is_answer_call {
+ // An abandoned answer call was never served, so it counts as neither a served
+ // request nor an error.
+ if is_answer_call && !is_abandoned(result) {
TOTAL_REQUESTS.fetch_add(1, Ordering::Relaxed);
let routed_attributes = [KeyValue::new("model", selected_model.to_string())];
if result.is_ok() {
@@ -273,7 +271,7 @@ pub(crate) fn record_llm_call(
}
}
}
- Err(error) => {
+ Err(error) if !is_abandoned(result) => {
span.record("error", tracing::field::display(error));
tracing::warn!(
target: TRACING_TARGET,
@@ -283,13 +281,13 @@ pub(crate) fn record_llm_call(
"model call failed"
);
}
+ Err(_) => {}
}
}
/// Records one published routing decision: the decision counter plus a
/// structured debug event carrying the decision's reasoning.
-pub(crate) fn record_decision(ctx: &Context, decision: &Decision) {
- let algorithm = algorithm_label(ctx);
+pub(crate) fn record_decision(algorithm: &str, decision: &Decision) {
let selected_model = decision.selected_model_id();
tracing::debug!(
target: TRACING_TARGET,
diff --git a/crates/protocol/README.md b/crates/protocol/README.md
index 56adc1701..f3dfd891b 100644
--- a/crates/protocol/README.md
+++ b/crates/protocol/README.md
@@ -20,7 +20,7 @@ serde_json = "1"
| Tools | [`ToolDefinition`], [`ToolChoice`], [`ToolCall`], [`ToolResult`] |
| Response | [`AggLlmResponse`], [`ResponseOutput`], [`Usage`], [`StopReason`] |
| Streaming | [`LlmResponse`], [`LlmResponseStream`], [`LlmResponseStreamEvent`], [`LlmResponseChunk`], [`ProviderStreamEvent`] |
-| Envelope | [`Context`], [`Request`], [`Response`], [`Metadata`] |
+| Envelope | [`Request`], [`Response`], [`Metadata`] |
| Routing I/O | [`Decision`], [`RoutedLlmClient`], [`LlmClientError`] |
| Wire identity | [`WireFormat`], [`FormatId`] |
diff --git a/crates/protocol/src/client.rs b/crates/protocol/src/client.rs
index 5abf5dcb6..cf137e84f 100644
--- a/crates/protocol/src/client.rs
+++ b/crates/protocol/src/client.rs
@@ -10,12 +10,10 @@
//! in libsy's orchestration crate — so a client crate that depends only on the
//! protocol can serve routed calls without pulling in the orchestrator.
-use std::sync::Arc;
-
use async_trait::async_trait;
use thiserror::Error;
-use crate::{Context, Request, Response};
+use crate::{Request, Response};
/// A boxed client-specific error preserved as the source of a routed call failure.
pub type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
@@ -186,11 +184,6 @@ pub trait RoutedLlmClient: Send + Sync {
/// [`decision.selected_model_id()`](Decision::selected_model_id), resolving it to the
/// provider model this client calls.
/// `request.llm_request.model` is the agent's original name, carried through for
- /// reference, not a call target. `ctx` carries the request's cross-cutting state.
- async fn call(
- &self,
- ctx: Context,
- request: Request,
- decision: Arc<Decision>,
- ) -> Result<Response, LlmClientError>;
+ /// reference, not a call target.
+ async fn call(&self, request: Request, decision: Decision) -> Result<Response, LlmClientError>;
}
diff --git a/crates/protocol/src/envelope.rs b/crates/protocol/src/envelope.rs
index 7139d9380..0986dca16 100644
--- a/crates/protocol/src/envelope.rs
+++ b/crates/protocol/src/envelope.rs
@@ -5,32 +5,6 @@
//! with the original provider payload and correlation [`Metadata`].
use crate::{LlmRequest, LlmResponse, Metadata};
-use std::collections::{HashMap, HashSet};
-
-/// Cross-cutting request context passed through algorithms and LLM clients.
-#[derive(Clone, Debug, Default, PartialEq, Eq)]
-pub struct Context {
- /// Caller-specific values propagated through the request.
- pub values: HashMap<String, String>,
- /// Targets this request must not route to. A caller may set these up front to keep a
- /// request off a given target; routing also adds any target that overflows its context
- /// window mid-request.
- excluded_targets: HashSet<String>,
-}
-
-impl Context {
- /// Excludes a target from this request, returning whether it was newly excluded.
- /// Re-excluding returns `false`, which is what bounds the overflow retry once every
- /// target has been tried.
- pub fn exclude_target(&mut self, target: impl Into<String>) -> bool {
- self.excluded_targets.insert(target.into())
- }
-
- /// Whether this request is barred from routing to `target`.
- pub fn is_excluded(&self, target: &str) -> bool {
- self.excluded_targets.contains(target)
- }
-}
/// Agentic-stack events fed to an algorithm out of band (e.g. tool results, budget
/// updates) — in libsy, via `Algorithm::process_signals`.
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index b4ce79d65..afc647fd7 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -16,7 +16,7 @@ use switchyard_libsy::{
};
use switchyard_llm_client::ClientRouter;
use switchyard_protocol::{
- AggLlmResponse, Context, Decision, LlmClientError, LlmResponse, Metadata, Request, Response,
+ AggLlmResponse, Decision, LlmClientError, LlmResponse, Metadata, Request, Response,
RoutedLlmClient,
};
@@ -33,9 +33,8 @@ struct PythonLlmClient {
impl RoutedLlmClient for PythonLlmClient {
async fn call(
&self,
- _ctx: Context,
request: Request,
- _decision: Arc<Decision>,
+ _decision: Decision,
) -> Result<Response, LlmClientError> {
let metadata = request.metadata;
let future = Python::attach(|py| {
@@ -250,15 +249,10 @@ impl PyAlgorithm {
metadata: headers.map(|headers| Metadata::from_headers(&headers)),
};
pyo3_async_runtimes::tokio::future_into_py(py, async move {
- let (decisions, response) = switchyard_llm_client::run(
- algorithm,
- client_router,
- Context::default(),
- request,
- None,
- )
- .await
- .map_err(py_libsy_error)?;
+ let (decisions, response) =
+ switchyard_llm_client::run(algorithm, client_router, request, None)
+ .await
+ .map_err(py_libsy_error)?;
let response = response
.llm_response
.into_agg()
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index fcd9cafdc..1f69e0b02 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -37,7 +37,7 @@ use parking_lot::Mutex;
use serde::Deserialize;
use serde_json::{Value, json};
use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver, TranslatingLlmClient};
-use switchyard_protocol::{Context, Decision, LlmClientError, Metadata, Request, Usage};
+use switchyard_protocol::{Decision, LlmClientError, Metadata, Request, Usage};
use tokio::net::{TcpListener, TcpSocket};
use tokio::task;
use tracing::{Instrument, Level};
@@ -693,18 +693,11 @@ async fn handle_llm_request(
state.stats.clone(),
state.routing_log.clone().zip(routing_log_context.clone()),
);
- let (trace, response) = match switchyard_llm_client::run(
- algorithm,
- client_router,
- Context::default(),
- request,
- Some(observer),
- )
- .await
- {
- Ok(result) => result,
- Err(error) => return algorithm_error(error),
- };
+ let (trace, response) =
+ match switchyard_llm_client::run(algorithm, client_router, request, Some(observer)).await {
+ Ok(result) => result,
+ Err(error) => return algorithm_error(error),
+ };
// Metrics, response body, and routing header all read the same decision, so
// the model they name can never disagree. An empty trace leaves the body with
// the id the upstream reported.
@@ -736,7 +729,7 @@ async fn handle_llm_request(
Err(error) => return server_error(error.to_string()),
};
if let Some(decision) = decision {
- attach_routing_headers(&mut response, decision.as_ref());
+ attach_routing_headers(&mut response, decision);
}
response
}