Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

refactor(libsy): make Decision a struct (#338)

Ayush Agarwal4fc554af5599
diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.md
index 1ce9fbf71..e17655b9d 100644
--- a/crates/libsy-llm-client/README.md
+++ b/crates/libsy-llm-client/README.md
@@ -159,7 +159,7 @@ async fn route(
switchyard_llm_client::run(algorithm, clients, Context::default(), request, None).await?;
Ok(trace
.last()
- .map(|decision| decision.selected_model().to_string())
+ .map(|decision| decision.selected_model_id().to_string())
.unwrap_or_default())
}
```
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index 54c9737b6..0bede8d71 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -511,9 +511,9 @@ impl RoutedLlmClient for TranslatingLlmClient {
&self,
ctx: Context,
request: Request,
- decision: std::sync::Arc<dyn Decision>,
+ decision: std::sync::Arc<Decision>,
) -> Result<Response> {
- let model_name = Some(decision.selected_model());
+ let model_name = Some(decision.selected_model_id());
self.call_rewrite_model(ctx, request, model_name).await
}
}
@@ -1721,7 +1721,7 @@ mod tests {
client.client = reqwest::Client::builder()
.timeout(std::time::Duration::from_millis(10))
.build()?;
- let decision: std::sync::Arc<dyn Decision> = std::sync::Arc::new(FixedDecision("gpt"));
+ let decision = std::sync::Arc::new(fixed_decision("gpt"));
let Err(error) = client
.call(Context::default(), request_for(None, false), decision)
@@ -1825,19 +1825,8 @@ mod tests {
Ok(())
}
- // Minimal `Decision` for driving the client through the `RoutedLlmClient` trait.
- struct FixedDecision(&'static str);
-
- impl Decision for FixedDecision {
- fn selected_model(&self) -> &str {
- self.0
- }
- fn reasoning(&self) -> Option<&str> {
- None
- }
- fn as_any(&self) -> &dyn std::any::Any {
- self
- }
+ fn fixed_decision(target: &str) -> Decision {
+ Decision::new(target, None, true)
}
// Exercises the `RoutedLlmClient` impl: `call` resolves the upstream model from the
@@ -1862,7 +1851,7 @@ mod tests {
.await;
let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
- let decision: std::sync::Arc<dyn Decision> = std::sync::Arc::new(FixedDecision("gpt"));
+ let decision = std::sync::Arc::new(fixed_decision("gpt"));
// Called through the trait; the request has no model, so "gpt" comes from the decision.
let response = client
.call(Context::default(), request_for(None, false), decision)
diff --git a/crates/libsy-llm-client/src/observation.rs b/crates/libsy-llm-client/src/observation.rs
index c6b955081..06352e25f 100644
--- a/crates/libsy-llm-client/src/observation.rs
+++ b/crates/libsy-llm-client/src/observation.rs
@@ -13,10 +13,8 @@ use switchyard_protocol::Usage;
pub struct LlmCallObservation {
/// Model selected for the completed call.
pub selected_model: String,
- /// Routing tier attached to the selected model, when present.
- pub tier: Option<String>,
- /// Whether this was the routed backend call rather than classifier or judge overhead.
- pub is_routed: bool,
+ /// Whether this call generated an answer rather than a routing verdict.
+ pub is_answer_call: bool,
/// Whether the call completed successfully.
pub is_success: bool,
/// Time spent waiting for the model call to resolve.
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index c3655a07b..2b8e017c4 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -41,7 +41,7 @@ pub async fn run(
ctx: Context,
request: Request,
observer: Option<RunObserver>,
-) -> Result<(Vec<Arc<dyn Decision>>, Response)> {
+) -> Result<(Vec<Arc<Decision>>, Response)> {
let algorithm_name = algorithm.name().to_string();
// The output from `serve` goes in here: when each successful routed call was in
// flight. Everything else the run spent time on is routing overhead.
@@ -110,13 +110,12 @@ impl RoutedCallWindows {
fields(
algorithm = algorithm_label(&call.get_routed().ctx),
switchyard.algorithm = algorithm_label(&call.get_routed().ctx),
- switchyard.routing.tier = tracing::field::Empty,
- selected_model = call.get_decision().selected_model(),
+ selected_model = call.get_decision().selected_model_id(),
otel.kind = "client",
- otel.name = %format_args!("chat {}", call.get_decision().selected_model()),
+ otel.name = %format_args!("chat {}", call.get_decision().selected_model_id()),
openinference.span.kind = "LLM",
gen_ai.operation.name = "chat",
- gen_ai.request.model = call.get_decision().selected_model(),
+ gen_ai.request.model = call.get_decision().selected_model_id(),
gen_ai.request.stream = tracing::field::Empty,
gen_ai.request.temperature = tracing::field::Empty,
gen_ai.request.top_p = tracing::field::Empty,
@@ -149,9 +148,6 @@ async fn serve(
) -> Result<()> {
let span = tracing::Span::current();
observability::record_gen_ai_request(&span, &call.get_routed().request.llm_request);
- if let Some(tier) = call.get_decision().routing_tier() {
- span.record("switchyard.routing.tier", tier);
- }
if let Some(session_id) = call
.get_routed()
.request
@@ -162,9 +158,8 @@ async fn serve(
span.record("gen_ai.conversation.id", session_id);
}
let routed = call.get_routed().clone();
- let target = routed.decision.selected_model().to_string();
- let tier = call.get_decision().routing_tier().map(str::to_string);
- let is_routed = call.get_decision().is_routed_call();
+ let target = routed.decision.selected_model_id().to_string();
+ let is_answer_call = call.get_decision().is_answer_call();
// Resolved before the clock starts: picking the client is Switchyard's work, not
// the provider's, so it belongs in the routing overhead.
let client = clients.route(&target);
@@ -184,9 +179,8 @@ async fn serve(
let result = observability::observe_client_call(result);
if let Some(observer) = observer {
observer(RunObservation::LlmCall(LlmCallObservation {
- selected_model: call.get_decision().selected_model().to_string(),
- tier,
- is_routed,
+ selected_model: call.get_decision().selected_model_id().to_string(),
+ is_answer_call,
is_success: result.is_ok(),
duration,
usage: result
@@ -196,7 +190,7 @@ async fn serve(
.map(|response| response.usage.clone()),
}));
}
- if is_routed && result.is_ok() {
+ if is_answer_call && result.is_ok() {
routed_calls.lock().record(started, ended);
}
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs
index 721b722d0..84630ee65 100644
--- a/crates/libsy-llm-client/tests/observability.rs
+++ b/crates/libsy-llm-client/tests/observability.rs
@@ -317,24 +317,6 @@ fn u64_gauge_value(snapshots: &[ResourceMetrics], name: &str) -> Option<u64> {
})
}
-/// Decision with a fixed model and reasoning string.
-struct StaticDecision {
- model: String,
- reasoning: String,
-}
-
-impl Decision for StaticDecision {
- fn selected_model(&self) -> &str {
- &self.model
- }
- fn reasoning(&self) -> Option<&str> {
- Some(&self.reasoning)
- }
- fn as_any(&self) -> &dyn std::any::Any {
- self
- }
-}
-
/// Client that answers every call with a fixed token [`Usage`].
struct UsageClient {
usage: Usage,
@@ -353,10 +335,10 @@ impl RoutedLlmClient for ClassifierClient {
&self,
_ctx: Context,
_request: Request,
- decision: Arc<dyn Decision>,
+ decision: Arc<Decision>,
) -> Result<Response, LlmClientError> {
- let model = decision.selected_model().to_string();
- let completion = if decision.is_routed_call() {
+ let model = decision.selected_model_id().to_string();
+ let completion = if decision.is_answer_call() {
tokio::time::sleep(self.routed_delay).await;
"routed response"
} else {
@@ -387,12 +369,12 @@ impl RoutedLlmClient for JudgeClient {
&self,
_ctx: Context,
_request: Request,
- decision: Arc<dyn Decision>,
+ decision: Arc<Decision>,
) -> Result<Response, LlmClientError> {
- if decision.is_routed_call() {
+ if decision.is_answer_call() {
return Ok(Response {
llm_response: LlmResponse::Agg(text_response(
- Some(decision.selected_model().to_string()),
+ Some(decision.selected_model_id().to_string()),
"routed response",
)),
metadata: None,
@@ -428,10 +410,10 @@ impl RoutedLlmClient for UsageClient {
&self,
_ctx: Context,
_request: Request,
- decision: Arc<dyn Decision>,
+ decision: Arc<Decision>,
) -> Result<Response, switchyard_protocol::LlmClientError> {
let mut response = text_response(
- Some(decision.selected_model().to_string()),
+ Some(decision.selected_model_id().to_string()),
"observed response",
);
response.id = Some("obs-response-1".to_string());
@@ -469,10 +451,11 @@ impl Algorithm for SingleCallAlgo {
.first()
.ok_or(LibsyError::NoTargets)?
.clone();
- let decision: Arc<dyn Decision> = Arc::new(StaticDecision {
- reasoning: format!("picked '{}'", target.semantic_name),
- model: target.semantic_name.clone(),
- });
+ let decision = Arc::new(Decision::new(
+ target.semantic_name.clone(),
+ Some(format!("picked '{}'", target.semantic_name)),
+ true,
+ ));
driver.info(ctx.clone(), decision.clone()).await?;
driver
.call_llm(RoutedRequest {
@@ -515,7 +498,7 @@ async fn run(
algorithm: Arc<dyn Algorithm>,
client: Arc<dyn RoutedLlmClient>,
request: Request,
-) -> switchyard_libsy::Result<(Vec<Arc<dyn Decision>>, Response)> {
+) -> switchyard_libsy::Result<(Vec<Arc<Decision>>, Response)> {
switchyard_llm_client::run(
algorithm,
ClientRouter::single(client),
@@ -877,7 +860,7 @@ async fn observed_run_reports_one_successful_routed_call() -> switchyard_libsy::
return Err(test_error("expected an LLM call observation"));
};
assert_eq!(observation.selected_model, MODEL);
- assert!(observation.is_routed);
+ assert!(observation.is_answer_call);
assert!(observation.is_success);
assert!(observation.usage.is_some());
assert!(matches!(
@@ -896,7 +879,7 @@ impl RoutedLlmClient for StreamingUsageClient {
&self,
_ctx: Context,
_request: Request,
- decision: Arc<dyn Decision>,
+ decision: Arc<Decision>,
) -> Result<Response, LlmClientError> {
let usage = Usage {
input_tokens: Some(13),
@@ -907,7 +890,7 @@ impl RoutedLlmClient for StreamingUsageClient {
let chunks = vec![Ok(LlmResponseStreamEvent::new(vec![
LlmResponseChunk::MessageStart {
id: Some("obs-stream-response".to_string()),
- model: Some(decision.selected_model().to_string()),
+ model: Some(decision.selected_model_id().to_string()),
},
LlmResponseChunk::Usage(usage),
LlmResponseChunk::MessageStop {
@@ -929,7 +912,7 @@ impl RoutedLlmClient for TimeoutClient {
&self,
_ctx: Context,
_request: Request,
- _decision: Arc<dyn Decision>,
+ _decision: Arc<Decision>,
) -> Result<Response, LlmClientError> {
Err(LlmClientError::Timeout {
source: Box::new(TestError("upstream timed out")),
@@ -1173,9 +1156,11 @@ async fn classifier_metrics_count_only_the_final_routed_call() -> switchyard_lib
let (trace, _response) = run(router, client, classifier_request()).await?;
- assert_eq!(
- trace.last().and_then(|decision| decision.routing_tier()),
- Some("weak")
+ assert!(
+ trace
+ .last()
+ .and_then(|decision| decision.reasoning())
+ .is_some_and(|reasoning| reasoning.contains("routing tier: weak"))
);
let snapshots = flushed_metrics(exporter, provider);
@@ -1192,11 +1177,7 @@ async fn classifier_metrics_count_only_the_final_routed_call() -> switchyard_lib
Some(1)
);
assert_eq!(
- u64_counter_value(
- &snapshots,
- "switchyard.requests",
- &[("model", "weak"), ("tier", "weak")],
- ),
+ u64_counter_value(&snapshots, "switchyard.requests", &[("model", "weak")],),
Some(1)
);
assert_eq!(
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index 60a7c8808..a290a0a12 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -51,38 +51,6 @@ const SESSION_STATE_TTL: Duration = Duration::from_secs(60 * 60);
/// Run the expired session cleanup code this often.
const SESSION_CLEANUP_INTERVAL: Duration = Duration::from_secs(60 * 60);
-/// The decision a fall-through run produces: the selected model plus a human-readable reason.
-pub struct FallThroughDecision {
- /// Target selected by the classifier cascade.
- pub selected_model: String,
- /// Human-readable explanation of the selection.
- pub reasoning: String,
- tier: Option<&'static str>,
- fallback_reason: Option<RoutingFallbackReason>,
-}
-
-impl Decision for FallThroughDecision {
- fn selected_model(&self) -> &str {
- &self.selected_model
- }
-
- fn routing_tier(&self) -> Option<&str> {
- self.tier
- }
-
- fn fallback_reason(&self) -> Option<RoutingFallbackReason> {
- self.fallback_reason
- }
-
- fn reasoning(&self) -> Option<&str> {
- Some(&self.reasoning)
- }
-
- fn as_any(&self) -> &dyn std::any::Any {
- self
- }
-}
-
/// Terminal classifier for a cascade whose classifiers may all abstain.
///
/// A classifier abstains when it cannot decide, which lets the next one try. The
@@ -298,20 +266,24 @@ where
from: &LlmTarget,
to: &LlmTarget,
reason: RoutingFallbackReason,
- ) -> Arc<dyn Decision> {
+ ) -> Arc<Decision> {
let failure = match reason {
RoutingFallbackReason::ContextWindow => "exceeded its context window",
RoutingFallbackReason::Unavailable => "was unavailable",
};
- Arc::new(FallThroughDecision {
- selected_model: to.semantic_name.clone(),
- reasoning: format!(
- "{} {failure}; fell back to {}",
- from.semantic_name, to.semantic_name,
- ),
- tier: deciding.routing_tier(&to.semantic_name),
- fallback_reason: Some(reason),
- })
+ Arc::new(Decision::new(
+ to.semantic_name.clone(),
+ Some(with_routing_tier(
+ format!(
+ "{} {failure}; fell back to {} (fallback reason: {})",
+ from.semantic_name,
+ to.semantic_name,
+ reason.as_str(),
+ ),
+ deciding.routing_tier(&to.semantic_name),
+ )),
+ true,
+ ))
}
/// Returns this request's retained state without holding the registry lock.
@@ -336,7 +308,7 @@ where
request: &mut Request,
) -> Result<(
LlmTarget,
- Arc<dyn Decision>,
+ Arc<Decision>,
Option<Response>,
Arc<dyn Classifier<S>>,
)> {
@@ -364,7 +336,7 @@ where
};
// 3. Resolve the target and publish the decision. When an excluded target sends
- // the request elsewhere, the tier and reasoning describe where it actually went.
+ // the request elsewhere, the reasoning describes where it actually went.
let target = self.targets.resolve_target(&score.target, ctx)?;
let reasoning = if target.semantic_name == score.target {
(self.decision_reason)(&self.name, &score)
@@ -374,16 +346,18 @@ where
score.target, target.semantic_name
)
};
- let decision: Arc<dyn Decision> = Arc::new(FallThroughDecision {
- selected_model: target.semantic_name.clone(),
- reasoning,
- tier: deciding.routing_tier(&target.semantic_name),
- fallback_reason: None,
- });
+ let decision: Arc<Decision> = Arc::new(Decision::new(
+ target.semantic_name.clone(),
+ Some(with_routing_tier(
+ reasoning,
+ deciding.routing_tier(&target.semantic_name),
+ )),
+ true,
+ ));
driver.info(ctx.clone(), decision.clone()).await?;
// 4. Post-decision replay: every processor sees the decision so stateful ones
- // can bind it, and may rewrite the outbound request (e.g. add a tier prompt).
+ // can bind it, and may rewrite the outbound request (e.g. add a target prompt).
for processor in &self.processors {
let event = Event::Decision {
request,
@@ -438,6 +412,14 @@ fn default_decision_reason(_name: &str, winner: &Score) -> String {
)
}
+// Routing tier is not part of Decision struct, so keeping it in reasoning for now. Remove it later if needed from reasoning.
+fn with_routing_tier(reasoning: String, tier: Option<&str>) -> String {
+ match tier {
+ Some(tier) => format!("{reasoning}; routing tier: {tier}"),
+ None => reasoning,
+ }
+}
+
#[async_trait]
impl<S> Algorithm for FallThrough<S>
where
@@ -481,11 +463,11 @@ mod tests {
/// Echoes the routed model name, capturing the request it was handed so a test can
/// assert on what actually reached the model.
fn capturing(into: Arc<Mutex<Option<Request>>>) -> impl Serve {
- move |decision: Arc<dyn Decision>, request: Request| {
+ move |decision: Arc<Decision>, request: Request| {
let into = Arc::clone(&into);
async move {
*into.lock() = Some(request);
- Ok(reply(decision.selected_model()))
+ Ok(reply(decision.selected_model_id()))
}
}
}
@@ -509,11 +491,11 @@ mod tests {
impl PromptRecorder {
fn serve(self: &Arc<Self>) -> impl Serve {
let recorder = Arc::clone(self);
- move |decision: Arc<dyn Decision>, request: Request| {
+ move |decision: Arc<Decision>, request: Request| {
let recorder = Arc::clone(&recorder);
async move {
*recorder.0.lock() = Some(RecordedCall {
- target: decision.selected_model().to_string(),
+ target: decision.selected_model_id().to_string(),
messages: request
.llm_request
.messages
@@ -527,7 +509,7 @@ mod tests {
.filter_map(|block| block.content.iter().find_map(text_of))
.collect(),
});
- Ok(reply(decision.selected_model()))
+ Ok(reply(decision.selected_model_id()))
}
}
}
@@ -644,7 +626,7 @@ mod tests {
router: &Arc<FallThrough<S>>,
request: Request,
serve: impl Serve,
- ) -> Result<(String, Vec<Arc<dyn Decision>>)>
+ ) -> Result<(String, Vec<Arc<Decision>>)>
where
S: Default + Send + 'static,
{
@@ -663,7 +645,7 @@ mod tests {
async fn run_turn<S>(
router: &Arc<FallThrough<S>>,
serve: impl Serve,
- ) -> Result<(String, Vec<Arc<dyn Decision>>)>
+ ) -> Result<(String, Vec<Arc<Decision>>)>
where
S: Default + Send + 'static,
{
@@ -674,7 +656,7 @@ mod tests {
async fn run_with(
router: FallThrough,
serve: impl Serve,
- ) -> Result<(String, Vec<Arc<dyn Decision>>)> {
+ ) -> Result<(String, Vec<Arc<Decision>>)> {
run_turn(&Arc::new(router), serve).await
}
@@ -689,10 +671,10 @@ mod tests {
overflowing: &'static [&'static str],
calls: Arc<Mutex<Vec<String>>>,
) -> impl Serve {
- move |decision: Arc<dyn Decision>, _request: Request| {
+ move |decision: Arc<Decision>, _request: Request| {
let calls = Arc::clone(&calls);
async move {
- let model = decision.selected_model().to_string();
+ let model = decision.selected_model_id().to_string();
calls.lock().push(model.clone());
if overflowing.contains(&model.as_str()) {
return Err(LlmClientError::ContextWindowExceeded {
@@ -711,10 +693,10 @@ mod tests {
unavailable: &'static [&'static str],
calls: Arc<Mutex<Vec<String>>>,
) -> impl Serve {
- move |decision: Arc<dyn Decision>, _request: Request| {
+ move |decision: Arc<Decision>, _request: Request| {
let calls = Arc::clone(&calls);
async move {
- let model = decision.selected_model().to_string();
+ let model = decision.selected_model_id().to_string();
calls.lock().push(model.clone());
if unavailable.contains(&model.as_str()) {
return Err(LlmClientError::UpstreamHttp {
@@ -888,15 +870,64 @@ mod tests {
for _ in 0..2 {
let (model, trace) = run_turn(&router, unavailable(&["weak"], calls.clone())).await?;
assert_eq!(model, "strong");
- assert_eq!(
- trace.last().and_then(|decision| decision.fallback_reason()),
- Some(RoutingFallbackReason::Unavailable)
+ assert!(
+ trace
+ .last()
+ .and_then(|decision| decision.reasoning())
+ .is_some_and(|reasoning| reasoning.contains("fallback reason: unavailable"))
);
}
assert_eq!(&*calls.lock(), &["weak", "strong", "weak", "strong"]);
Ok(())
}
+ #[tokio::test]
+ async fn fallback_decision_preserves_tier_and_cause_in_reasoning() -> Result<()> {
+ struct TieredClassifier;
+
+ #[async_trait]
+ impl Classifier for TieredClassifier {
+ fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> {
+ match selected_model_id {
+ "weak" => Some("weak"),
+ "strong" => Some("strong"),
+ _ => None,
+ }
+ }
+
+ async fn score(
+ &self,
+ _state: &mut (),
+ _request: &mut Request,
+ _driver: Option<&Driver>,
+ ) -> Result<(Classification, Option<Response>)> {
+ Ok((Classification::Scores(vec![score("weak", 1.0)]), None))
+ }
+ }
+
+ let router = FallThrough::<()>::new(target_set(&["weak", "strong"]))
+ .with_classifier(Arc::new(TieredClassifier));
+ let (_, trace) = run_with(router, unavailable(&["weak"], Arc::default())).await?;
+
+ assert_eq!(trace.len(), 2);
+ assert_eq!(trace[0].selected_model_id(), "weak");
+ assert!(trace[0].is_answer_call());
+ assert!(
+ trace[0]
+ .reasoning()
+ .is_some_and(|reasoning| reasoning.contains("routing tier: weak"))
+ );
+
+ let fallback = &trace[1];
+ assert_eq!(fallback.selected_model_id(), "strong");
+ assert!(fallback.is_answer_call());
+ assert!(fallback.reasoning().is_some_and(|reasoning| {
+ reasoning.contains("fallback reason: unavailable")
+ && reasoning.contains("routing tier: strong")
+ }));
+ Ok(())
+ }
+
#[tokio::test]
async fn overflowing_targets_are_retried_until_one_fits() -> Result<()> {
let router = FallThrough::<()>::new(target_set(&["weak", "mid", "strong"]))
@@ -968,7 +999,7 @@ mod tests {
.map_err(|error| LibsyError::external("aggregating fall-through response", error))?;
assert_eq!(text, "strong");
- assert_eq!(trace[0].selected_model(), "strong");
+ assert_eq!(trace[0].selected_model_id(), "strong");
assert!(
trace[0]
.reasoning()
@@ -984,7 +1015,7 @@ mod tests {
let (model, trace) = run_with(router, echo()).await?;
assert_eq!(model, "strong");
assert_eq!(trace.len(), 1);
- assert_eq!(trace[0].selected_model(), "strong");
+ assert_eq!(trace[0].selected_model_id(), "strong");
Ok(())
}
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index d5be7492f..1df07ae06 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -9,7 +9,7 @@ use std::sync::Arc;
use async_trait::async_trait;
use serde::{Deserialize, Deserializer};
use serde_json::Value;
-use switchyard_protocol::{ContentBlock, Message, Role, SimpleDecision};
+use switchyard_protocol::{ContentBlock, Decision, Message, Role};
use super::fall_through::{DefaultTarget, FallThrough};
use super::util::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS;
@@ -481,12 +481,12 @@ struct EscalationClassifier {
#[async_trait]
impl Classifier<State> for EscalationClassifier {
- fn routing_tier(&self, selected_model: &str) -> Option<&'static str> {
+ fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> {
if self.capable.semantic_name == self.efficient.semantic_name {
None
- } else if selected_model == self.capable.semantic_name {
+ } else if selected_model_id == self.capable.semantic_name {
Some("strong")
- } else if selected_model == self.efficient.semantic_name {
+ } else if selected_model_id == self.efficient.semantic_name {
Some("weak")
} else {
None
@@ -521,10 +521,11 @@ impl Classifier<State> for EscalationClassifier {
let efficient_response = match driver
.call_llm(RoutedRequest {
request: request.clone(),
- decision: Arc::new(SimpleDecision {
- selected_model: self.efficient.semantic_name.clone(),
- reasoning: Some("escalation classifier: efficient tier".into()),
- }),
+ decision: Arc::new(Decision::new(
+ self.efficient.semantic_name.clone(),
+ Some("escalation classifier: efficient tier".into()),
+ true,
+ )),
ctx: Context::default(),
})
.await
@@ -891,12 +892,12 @@ impl LlmTaskClassifier {
#[async_trait]
impl Classifier<State> for TaskClassifier {
- fn routing_tier(&self, selected_model: &str) -> Option<&'static str> {
+ fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> {
if self.efficient_target == self.capable_target {
None
- } else if selected_model == self.efficient_target {
+ } else if selected_model_id == self.efficient_target {
Some("weak")
- } else if selected_model == self.capable_target {
+ } else if selected_model_id == self.capable_target {
Some("strong")
} else {
None
@@ -915,8 +916,8 @@ impl Classifier<State> for TaskClassifier {
#[async_trait]
impl Classifier<State> for LlmTaskClassifier {
- fn routing_tier(&self, selected_model: &str) -> Option<&'static str> {
- self.inner.routing_tier(selected_model)
+ fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> {
+ self.inner.routing_tier(selected_model_id)
}
async fn score(
@@ -1006,6 +1007,7 @@ mod tests {
#[derive(Default)]
struct Recorder {
calls: Mutex<Vec<String>>,
+ call_roles: Mutex<Vec<(String, bool)>>,
judge_max_output_tokens: Mutex<Vec<Option<u64>>>,
judge_system_prompts: Mutex<Vec<String>>,
}
@@ -1015,6 +1017,10 @@ mod tests {
self.calls.lock().clone()
}
+ fn call_roles(&self) -> Vec<(String, bool)> {
+ self.call_roles.lock().clone()
+ }
+
fn judge_max_output_tokens(&self) -> Vec<Option<u64>> {
self.judge_max_output_tokens.lock().clone()
}
@@ -1025,11 +1031,15 @@ mod tests {
fn serve(self: &Arc<Self>) -> impl Serve {
let recorder = Arc::clone(self);
- move |decision: Arc<dyn Decision>, request: Request| {
+ move |decision: Arc<Decision>, request: Request| {
let recorder = Arc::clone(&recorder);
async move {
- let model = decision.selected_model().to_string();
+ let model = decision.selected_model_id().to_string();
recorder.calls.lock().push(model.clone());
+ recorder
+ .call_roles
+ .lock()
+ .push((model.clone(), decision.is_answer_call()));
let completion = if model == "judge" {
recorder
.judge_max_output_tokens
@@ -1065,8 +1075,8 @@ mod tests {
/// The judge times out; every other target answers normally.
fn unreachable_judge() -> impl Serve {
- |decision: Arc<dyn Decision>, request: Request| async move {
- let model = decision.selected_model().to_string();
+ |decision: Arc<Decision>, request: Request| async move {
+ let model = decision.selected_model_id().to_string();
if model == "judge" {
return Err(LlmClientError::Timeout {
source: Box::new(std::io::Error::other("judge unreachable")),
@@ -1136,7 +1146,7 @@ mod tests {
)
.await?;
- assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));
+ assert_eq!(trace.last().map(|d| d.selected_model_id()), Some("capable"));
assert_eq!(
response.llm_response.as_agg().map(completion_text),
Some("answer from capable".to_string())
@@ -1169,6 +1179,15 @@ mod tests {
recorder.calls(),
vec!["judge", "efficient", "judge", "efficient"]
);
+ assert_eq!(
+ recorder.call_roles(),
+ vec![
+ ("judge".to_string(), false),
+ ("efficient".to_string(), true),
+ ("judge".to_string(), false),
+ ("efficient".to_string(), true),
+ ]
+ );
Ok(())
}
@@ -1818,8 +1837,8 @@ mod tests {
/// Serves the judge target from `judge` and every other target from `model`, each with
/// its next queued reply.
fn queued(model: Arc<Queue>, judge: Arc<Queue>) -> impl Serve {
- move |decision: Arc<dyn Decision>, request: Request| {
- let queue = if decision.selected_model() == "judge" {
+ move |decision: Arc<Decision>, request: Request| {
+ let queue = if decision.selected_model_id() == "judge" {
Arc::clone(&judge)
} else {
Arc::clone(&model)
@@ -1869,7 +1888,16 @@ mod tests {
.await?;
// The efficient model is the serving target, and the response comes from its call.
- assert_eq!(trace.last().map(|d| d.selected_model()), Some("efficient"));
+ assert_eq!(
+ trace.last().map(|d| d.selected_model_id()),
+ Some("efficient")
+ );
+ assert!(
+ trace
+ .last()
+ .and_then(|decision| decision.reasoning())
+ .is_some_and(|reasoning| reasoning.contains("routing tier: weak"))
+ );
assert_eq!(
response.llm_response.as_agg().map(completion_text),
Some("efficient answer".to_string())
@@ -1925,7 +1953,13 @@ mod tests {
)
.await?;
- assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));
+ assert_eq!(trace.last().map(|d| d.selected_model_id()), Some("capable"));
+ assert!(
+ trace
+ .last()
+ .and_then(|decision| decision.reasoning())
+ .is_some_and(|reasoning| reasoning.contains("routing tier: strong"))
+ );
assert_eq!(
response.llm_response.as_agg().map(completion_text),
Some("capable answer".to_string())
@@ -1957,7 +1991,7 @@ mod tests {
)
.await?;
- assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));
+ assert_eq!(trace.last().map(|d| d.selected_model_id()), Some("capable"));
Ok(())
}
@@ -1969,10 +2003,10 @@ mod tests {
let router = escalation_router()?;
// Efficient overflows, capable answers, and the judge must never be called.
- let serve = |decision: Arc<dyn Decision>, _request: Request| async move {
- match decision.selected_model() {
+ let serve = |decision: Arc<Decision>, _request: Request| async move {
+ match decision.selected_model_id() {
"efficient" => Err(LlmClientError::ContextWindowExceeded {
- model: decision.selected_model().to_string(),
+ model: decision.selected_model_id().to_string(),
message: "prompt is too long".to_string(),
}),
"judge" => panic!("the judge must not be consulted when efficient overflows"),
@@ -1983,7 +2017,7 @@ mod tests {
let (trace, response) =
test_drive(router, Context::default(), classify_request(), serve).await?;
- assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));
+ assert_eq!(trace.last().map(|d| d.selected_model_id()), Some("capable"));
assert_eq!(
response.llm_response.as_agg().map(completion_text),
Some("capable answer".to_string())
diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rs
index 853c297a1..973f7e0b5 100644
--- a/crates/libsy/src/algorithms/noop.rs
+++ b/crates/libsy/src/algorithms/noop.rs
@@ -16,23 +16,6 @@ use switchyard_protocol::{Context, Decision};
/// Test helper that returns a hard-coded response without routing or model I/O.
pub struct Noop {}
-/// Test decision carrying the inbound model or a fixed placeholder.
-pub struct NoopDecision {
- model: String,
-}
-
-impl Decision for NoopDecision {
- fn selected_model(&self) -> &str {
- &self.model
- }
- fn reasoning(&self) -> Option<&str> {
- None
- }
- fn as_any(&self) -> &dyn std::any::Any {
- self
- }
-}
-
#[async_trait::async_trait]
impl Algorithm for Noop {
fn name(&self) -> &str {
@@ -49,9 +32,11 @@ impl Algorithm for Noop {
.requested_model()
.unwrap_or("switchyard/noop")
.to_string();
- let decision: Arc<dyn Decision> = Arc::new(NoopDecision {
- model: model.clone(),
- });
+ let decision: Arc<Decision> = Arc::new(Decision::new(
+ model.clone(),
+ Some("noop returned its synthetic response".to_string()),
+ true,
+ ));
driver.info(ctx, decision.clone()).await?;
let llm_response = LlmResponse::Agg(AggLlmResponse {
@@ -101,7 +86,8 @@ mod tests {
let Some(decision) = decisions.first() else {
panic!("Expected exactly one Decision");
};
- assert_eq!(decision.selected_model(), TEST_MODEL);
+ assert_eq!(decision.selected_model_id(), TEST_MODEL);
+ assert!(decision.is_answer_call());
assert_eq!(response.selected_model(), Some(TEST_MODEL));
Ok(())
}
diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rs
index c8c74faeb..5a2a93284 100644
--- a/crates/libsy/src/algorithms/passthrough.rs
+++ b/crates/libsy/src/algorithms/passthrough.rs
@@ -23,24 +23,6 @@ impl Passthrough {
}
}
-/// Decision emitted before [`Passthrough`] calls its configured target.
-pub struct PassthroughDecision {
- model_id: String,
-}
-
-impl Decision for PassthroughDecision {
- fn selected_model(&self) -> &str {
- &self.model_id
- }
- fn reasoning(&self) -> Option<&str> {
- // There is no decision to take, so no reasoning
- None
- }
- fn as_any(&self) -> &dyn std::any::Any {
- self
- }
-}
-
#[async_trait::async_trait]
impl Algorithm for Passthrough {
fn name(&self) -> &str {
@@ -53,9 +35,14 @@ impl Algorithm for Passthrough {
driver: Driver,
request: Request,
) -> Result<Response> {
- let decision: Arc<dyn Decision> = Arc::new(PassthroughDecision {
- model_id: self.target.semantic_name.clone(),
- });
+ let decision: Arc<Decision> = Arc::new(Decision::new(
+ self.target.semantic_name.clone(),
+ Some(format!(
+ "passthrough selected target '{}'",
+ self.target.semantic_name
+ )),
+ true,
+ ));
driver.info(ctx.clone(), decision.clone()).await?;
driver
.call_llm(RoutedRequest {
@@ -98,7 +85,8 @@ mod tests {
MODEL_ID
);
assert_eq!(trace.len(), 1);
- assert_eq!(trace[0].selected_model(), MODEL_ID);
+ assert_eq!(trace[0].selected_model_id(), MODEL_ID);
+ assert!(trace[0].is_answer_call());
Ok(())
}
}
diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rs
index 5a0c3243c..d724a8de3 100644
--- a/crates/libsy/src/algorithms/rand.rs
+++ b/crates/libsy/src/algorithms/rand.rs
@@ -15,15 +15,12 @@ use rand::SeedableRng;
use rand::distr::{Distribution, weighted::WeightedIndex};
use rand::rngs::StdRng;
-use crate::algorithms::fall_through::{FallThrough, FallThroughDecision};
+use crate::algorithms::fall_through::FallThrough;
use crate::core::algorithm::{Algorithm, Driver, LlmTargetSet};
use crate::core::classifier::{Classification, Classifier, Score};
use crate::{LibsyError, Result};
use switchyard_protocol::{Context, Request, Response};
-/// Compatibility name for the decision produced by [`Random`].
-pub type RandomDecision = FallThroughDecision;
-
/// Stateless weighted classifier used by random fall-through routing.
pub struct RandomClassifier {
targets: Vec<String>,
@@ -251,7 +248,7 @@ mod tests {
"only/model"
);
assert_eq!(trace.len(), 1);
- assert_eq!(trace[0].selected_model(), "only/model");
+ assert_eq!(trace[0].selected_model_id(), "only/model");
Ok(())
}
@@ -272,7 +269,7 @@ mod tests {
names.contains(&selected.as_str()),
"selected {selected} not in target set"
);
- assert_eq!(trace[0].selected_model(), selected.as_str());
+ assert_eq!(trace[0].selected_model_id(), selected.as_str());
}
Ok(())
}
@@ -429,25 +426,19 @@ mod tests {
}
#[tokio::test]
- async fn decision_is_inspectable_and_downcasts() -> Result<()> {
+ async fn decision_is_inspectable() -> Result<()> {
let algorithm = shared_algorithm(&["only/model"])?;
let (trace, _) = test_drive(algorithm, Context::default(), request(), echo()).await?;
let decision = &trace[0];
- assert_eq!(decision.selected_model(), "only/model");
+ assert_eq!(decision.selected_model_id(), "only/model");
assert!(
decision
.reasoning()
.unwrap_or_default()
.contains("only/model")
);
- let concrete = decision
- .as_any()
- .downcast_ref::<RandomDecision>()
- .ok_or_else(|| LibsyError::AlgorithmError {
- message: "decision did not downcast to RandomDecision".to_string(),
- })?;
- assert_eq!(concrete.selected_model, "only/model");
+ assert!(decision.is_answer_call());
Ok(())
}
}
diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs
index a244b4bbd..5091d1d3f 100644
--- a/crates/libsy/src/algorithms/stage.rs
+++ b/crates/libsy/src/algorithms/stage.rs
@@ -45,8 +45,8 @@ struct SourceStamp {
#[async_trait]
impl Classifier<State> for SourceStamp {
- fn routing_tier(&self, selected_model: &str) -> Option<&'static str> {
- self.inner.routing_tier(selected_model)
+ fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> {
+ self.inner.routing_tier(selected_model_id)
}
async fn score(
@@ -355,6 +355,7 @@ mod tests {
struct Call {
target: String,
messages: Vec<String>,
+ is_answer_call: bool,
}
/// Records what each target receives.
@@ -369,7 +370,7 @@ mod tests {
self.calls
.lock()
.iter()
- .filter(|call| call.target != JUDGE)
+ .filter(|call| call.is_answer_call)
.cloned()
.collect()
}
@@ -378,10 +379,10 @@ mod tests {
/// back so the fallback classifier has an answer without a real model.
fn serve(self: &Arc<Self>) -> impl Serve {
let recorder = Arc::clone(self);
- move |decision: Arc<dyn Decision>, request: Request| {
+ move |decision: Arc<Decision>, request: Request| {
let recorder = Arc::clone(&recorder);
async move {
- let target = decision.selected_model().to_string();
+ let target = decision.selected_model_id().to_string();
recorder.calls.lock().push(Call {
target: target.clone(),
messages: request
@@ -390,6 +391,7 @@ mod tests {
.iter()
.filter_map(|message| message.text_content("|"))
.collect(),
+ is_answer_call: decision.is_answer_call(),
});
let completion = if target == JUDGE {
let p_solve = *recorder.judge_p_solve.lock();
@@ -527,7 +529,7 @@ mod tests {
let recorder = Arc::new(Recorder::default());
let router = recording_router(config_with_judge(&recorder, 0.1))?;
- test_drive(
+ let (trace, _) = test_drive(
router.clone(),
Context::default(),
turn_request(false),
@@ -535,11 +537,26 @@ mod tests {
)
.await?;
+ let calls = recorder.calls.lock();
assert!(
- recorder.calls.lock().iter().any(|c| c.target == JUDGE),
- "the judge should be consulted on an undecided turn"
+ calls
+ .iter()
+ .any(|call| call.target == JUDGE && !call.is_answer_call),
+ "the judge should be recorded as a routing side call"
+ );
+ assert!(
+ calls
+ .iter()
+ .any(|call| call.target == "strong" && call.is_answer_call),
+ "the selected target should be recorded as an answer call"
+ );
+ drop(calls);
+ assert!(
+ trace
+ .last()
+ .and_then(|decision| decision.reasoning())
+ .is_some_and(|reasoning| reasoning.contains("routing tier: strong"))
);
- assert_eq!(recorder.routed()[0].target, "strong");
Ok(())
}
diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rs
index da07eb7b4..6bcbb6815 100644
--- a/crates/libsy/src/algorithms/util/affinity.rs
+++ b/crates/libsy/src/algorithms/util/affinity.rs
@@ -127,7 +127,7 @@ where
if let Event::Decision { request, decision } = event
&& let Some(key) = self.affinity_key(request)
{
- let model = decision.selected_model();
+ let model = decision.selected_model_id();
let mut assignments = self.assignments.lock();
if self.should_latch(model) && !assignments.contains_key(&key) {
evict_if_full(&mut assignments);
@@ -215,21 +215,8 @@ mod tests {
/// Boxed, thread-safe error type keeping the test helpers ergonomic.
type BoxErr = Box<dyn std::error::Error + Send + Sync>;
- /// A decision that reports a fixed selected model.
- struct FixedDecision(&'static str);
-
- impl Decision for FixedDecision {
- fn selected_model(&self) -> &str {
- self.0
- }
-
- fn reasoning(&self) -> Option<&str> {
- None
- }
-
- fn as_any(&self) -> &dyn std::any::Any {
- self
- }
+ fn fixed_decision(target: &str) -> Decision {
+ Decision::new(target, None, true)
}
fn request(metadata: Metadata) -> Request {
@@ -289,12 +276,13 @@ mod tests {
request: &mut Request,
model: &'static str,
) -> Result<(), BoxErr> {
+ let decision = fixed_decision(model);
router
.process(
state,
Event::Decision {
request,
- decision: &FixedDecision(model),
+ decision: &decision,
},
)
.await?;
@@ -562,7 +550,7 @@ mod tests {
&mut state,
Event::Decision {
request: &mut first,
- decision: &FixedDecision("model-a"),
+ decision: &fixed_decision("model-a"),
},
)
.await?;
@@ -587,7 +575,7 @@ mod tests {
&mut state,
Event::Decision {
request: &mut unkeyed,
- decision: &FixedDecision("model-a"),
+ decision: &fixed_decision("model-a"),
},
)
.await?;
@@ -611,7 +599,7 @@ mod tests {
&mut state,
Event::Decision {
request: &mut second,
- decision: &FixedDecision("model-b"),
+ decision: &fixed_decision("model-b"),
},
)
.await?;
@@ -620,7 +608,7 @@ mod tests {
&mut state,
Event::Decision {
request: &mut first,
- decision: &FixedDecision("model-a"),
+ decision: &fixed_decision("model-a"),
},
)
.await?;
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
index dad60c31d..e4a75a018 100644
--- a/crates/libsy/src/algorithms/util/llm_judge.rs
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -229,9 +229,11 @@ where
let response = driver
.call_llm(RoutedRequest {
request: self.judge.build_request(state, request),
- decision: Arc::new(JudgeDecision {
- model: self.target.semantic_name.to_string(),
- }),
+ decision: Arc::new(Decision::new(
+ self.target.semantic_name.to_string(),
+ Some("llm judge consultation".to_string()),
+ false,
+ )),
ctx: Context::default(),
})
.await
@@ -324,28 +326,6 @@ fn parse_json_verdict<T: DeserializeOwned>(response: &AggLlmResponse) -> Result<
})
}
-struct JudgeDecision {
- model: String,
-}
-
-impl Decision for JudgeDecision {
- fn selected_model(&self) -> &str {
- &self.model
- }
-
- fn is_routed_call(&self) -> bool {
- false
- }
-
- fn reasoning(&self) -> Option<&str> {
- Some("llm judge consultation")
- }
-
- fn as_any(&self) -> &dyn std::any::Any {
- self
- }
-}
-
fn strip_json_fence(text: &str) -> &str {
let Some(rest) = text.strip_prefix("```") else {
return text;
diff --git a/crates/libsy/src/algorithms/util/prompts.rs b/crates/libsy/src/algorithms/util/prompts.rs
index 9b82dadc9..9d45e65db 100644
--- a/crates/libsy/src/algorithms/util/prompts.rs
+++ b/crates/libsy/src/algorithms/util/prompts.rs
@@ -117,7 +117,7 @@ impl<S: Send> Processor<S> for SystemPromptProcessor {
let Event::Decision { request, decision } = event else {
return Ok(());
};
- let Some(prompt) = self.prompts.get(decision.selected_model()) else {
+ let Some(prompt) = self.prompts.get(decision.selected_model_id()) else {
return Ok(());
};
// Ahead of the client's own instructions, so this framing is what the
@@ -139,7 +139,7 @@ impl<S: Send> Processor<S> for SystemPromptProcessor {
#[cfg(test)]
mod tests {
use super::*;
- use switchyard_protocol::{LlmRequest, ToolResult, text_request};
+ use switchyard_protocol::{Decision, LlmRequest, ToolResult, text_request};
const NOTE: &str = "recovering from an error";
const STRONG_PROMPT: &str = "diagnose before you edit";
@@ -241,20 +241,6 @@ mod tests {
);
}
- /// A decision routed to `target`.
- struct RoutedTo(&'static str);
- impl switchyard_protocol::Decision for RoutedTo {
- fn selected_model(&self) -> &str {
- self.0
- }
- fn reasoning(&self) -> Option<&str> {
- None
- }
- fn as_any(&self) -> &dyn std::any::Any {
- self
- }
- }
-
/// The instruction text the request carries.
fn instructions(request: &Request) -> Vec<String> {
request
@@ -279,12 +265,13 @@ mod tests {
},
..Request::default()
};
+ let decision = Decision::new(target, None, true);
processor
.process(
&mut (),
Event::Decision {
request: &mut request,
- decision: &RoutedTo(target),
+ decision: &decision,
},
)
.await?;
@@ -339,13 +326,14 @@ mod tests {
text: "you are a coding agent".to_string(),
}],
});
+ let decision = Decision::new("strong", None, true);
processor
.process(
&mut (),
Event::Decision {
request: &mut request,
- decision: &RoutedTo("strong"),
+ decision: &decision,
},
)
.await?;
diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs
index 24e3deded..3105c1102 100644
--- a/crates/libsy/src/algorithms/util/stage.rs
+++ b/crates/libsy/src/algorithms/util/stage.rs
@@ -476,8 +476,8 @@ impl StageClassifier {
#[async_trait]
impl Classifier<State> for StageClassifier {
- fn routing_tier(&self, selected_model: &str) -> Option<&'static str> {
- self.targets.label_for(selected_model)
+ fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> {
+ self.targets.label_for(selected_model_id)
}
async fn score(
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index f888aad54..73270a7cb 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -41,17 +41,17 @@ pub type StepStream = Pin<Box<dyn Stream<Item = Result<Step>> + Send>>;
/// A request paired with the routing [`Decision`] that produced it — the offload
/// payload a host reads (via [`CallLlmRequest::get_routed`]) to serve the call.
///
-/// The two model identifiers live in separate, unambiguous places: the model to
-/// call is [`decision.selected_model()`](Decision::selected_model), while
+/// The selected model and inbound route name live in separate, unambiguous places: the model
+/// identifier is [`decision.selected_model_id()`](Decision::selected_model_id), while
/// `request.llm_request.model` is the *inbound* name the agent asked for (libsy
-/// never overwrites it). A client maps `selected_model()` to the provider model
+/// never overwrites it). A client maps `selected_model_id()` to the provider model
/// id it hits.
#[derive(Clone)]
pub struct RoutedRequest {
/// The request to serve; its `model` is the agent's original name.
pub request: Request,
- /// The routing decision behind this call; `selected_model()` is the model to hit.
- pub decision: Arc<dyn Decision>,
+ /// The routing decision behind this call; `selected_model_id()` identifies the model to hit.
+ pub decision: Arc<Decision>,
/// The request's cross-cutting context, carried through the offload so the host
/// serving the call can pass it to its client.
pub ctx: Context,
@@ -70,7 +70,7 @@ pub struct CallLlmRequest {
}
impl CallLlmRequest {
- /// The routed request the host should serve; its `decision.selected_model()` names
+ /// The routed request the host should serve; its `decision.selected_model_id()` names
/// the model to hit.
pub fn get_routed(&self) -> &RoutedRequest {
&self.routed
@@ -81,8 +81,8 @@ impl CallLlmRequest {
&self.get_routed().request
}
- /// The decision that led to this call — its `selected_model()` is the model to hit.
- pub fn get_decision(&self) -> &dyn Decision {
+ /// The decision that led to this call; its `selected_model_id()` identifies the model to hit.
+ pub fn get_decision(&self) -> &Decision {
self.get_routed().decision.as_ref()
}
@@ -133,7 +133,7 @@ impl Driver {
skip_all,
fields(
algorithm = observability::algorithm_label(&routed.ctx),
- selected_model = routed.decision.selected_model(),
+ selected_model = routed.decision.selected_model_id(),
openinference.span.kind = "CHAIN",
outcome = tracing::field::Empty,
error = tracing::field::Empty,
@@ -145,9 +145,8 @@ impl Driver {
)]
pub async fn call_llm(&self, routed: RoutedRequest) -> Result<Response> {
let algorithm = observability::algorithm_label(&routed.ctx).to_string();
- let selected_model = routed.decision.selected_model().to_string();
- let tier = routed.decision.routing_tier().map(str::to_string);
- let is_routed = routed.decision.is_routed_call();
+ let decision = Arc::clone(&routed.decision);
+ let is_answer_call = decision.is_answer_call();
let started = Instant::now();
let (reply, response) = oneshot::channel::<Result<Response>>();
let call = CallLlmRequest { routed, reply };
@@ -164,9 +163,8 @@ impl Driver {
let elapsed = started.elapsed();
observability::record_llm_call(
&algorithm,
- &selected_model,
- tier.as_deref(),
- is_routed,
+ decision.selected_model_id(),
+ is_answer_call,
elapsed,
&result,
&tracing::Span::current(),
@@ -177,7 +175,7 @@ impl Driver {
/// Publish a routing [`Decision`] as a [`Step::Decision`] on the stream.
/// Each successfully published decision is counted and logged with its
/// reasoning; a decision the stream never accepted is not recorded.
- pub async fn info(&self, ctx: Context, decision: Arc<dyn Decision>) -> Result<()> {
+ pub async fn info(&self, ctx: Context, decision: Arc<Decision>) -> Result<()> {
self.step_tx
.send(Ok(Step::Decision(decision.clone())))
.await
@@ -205,7 +203,7 @@ pub enum Step {
CallLlm(Box<CallLlmRequest>),
/// A routing decision the algorithm made, published via [`Driver::info`] as it
/// happens (rather than collected into a trace returned at the end).
- Decision(Arc<dyn Decision>),
+ Decision(Arc<Decision>),
/// The algorithm finished with its final response — the last step of a run.
ReturnToAgent(Box<Response>),
}
@@ -227,7 +225,7 @@ pub async fn drive<F, Fut>(
ctx: Context,
request: Request,
serve: F,
-) -> Result<(Vec<Arc<dyn Decision>>, Response)>
+) -> Result<(Vec<Arc<Decision>>, Response)>
where
F: Fn(CallLlmRequest) -> Fut,
Fut: Future<Output = Result<()>>,
@@ -235,7 +233,7 @@ where
let stream = algorithm.run_stream(ctx, request);
tokio::pin!(stream);
- let mut trace: Vec<Arc<dyn Decision>> = Vec::new();
+ let mut trace: Vec<Arc<Decision>> = Vec::new();
let mut in_flight = futures::stream::FuturesUnordered::new();
let mut final_response: Option<Response> = None;
@@ -275,8 +273,8 @@ impl Drop for AbortOnDrop {
}
/// A named routing target an algorithm routes by. Serving its calls is the stream
-/// consumer's concern: the target's name reaches the consumer as
-/// `decision.selected_model()` on the offloaded [`RoutedRequest`].
+/// consumer's concern: the selected identifier reaches the consumer as
+/// `decision.selected_model_id()` on the offloaded [`RoutedRequest`].
#[derive(Clone)]
pub struct LlmTarget {
/// The routing label an algorithm selects this target by — a logical tier like
@@ -480,12 +478,12 @@ pub(crate) async fn call_llm_with_fallback(
driver: &Driver,
targets: &LlmTargetSet,
mut target: LlmTarget,
- mut decision: Arc<dyn Decision>,
+ mut decision: Arc<Decision>,
request: Request,
identity: Option<&RoutingIdentity>,
evictions: &SessionEvictions,
target_unavailable: impl Fn(&Request, &str),
- fallback_decision: impl Fn(&LlmTarget, &LlmTarget, RoutingFallbackReason) -> Arc<dyn Decision>,
+ fallback_decision: impl Fn(&LlmTarget, &LlmTarget, RoutingFallbackReason) -> Arc<Decision>,
) -> Result<Response> {
loop {
let result = driver
@@ -695,24 +693,13 @@ mod tests {
);
}
- /// Trivial decision + algo used only to exercise the orchestrator: calls the
- /// first target and returns its response with a one-item trace.
- struct TestDecision {
- model: String,
- }
-
- impl Decision for TestDecision {
- fn selected_model(&self) -> &str {
- &self.model
- }
- fn reasoning(&self) -> Option<&str> {
- None
- }
- fn as_any(&self) -> &dyn std::any::Any {
- self
- }
+ /// Build a routed decision for orchestration tests.
+ fn test_decision(selected_model_id: String) -> Arc<Decision> {
+ Arc::new(Decision::new(selected_model_id, None, true))
}
+ /// Trivial algo used only to exercise the orchestrator: calls the first target
+ /// and returns its response with a one-item trace.
struct TestAlgo {
target_set: LlmTargetSet,
}
@@ -735,9 +722,7 @@ mod tests {
.first()
.ok_or(LibsyError::NoTargets)?
.clone();
- let decision: Arc<dyn Decision> = Arc::new(TestDecision {
- model: target.semantic_name.clone(),
- });
+ let decision = test_decision(target.semantic_name.clone());
driver.info(ctx.clone(), decision.clone()).await?;
driver
.call_llm(RoutedRequest {
@@ -775,9 +760,7 @@ mod tests {
fn routed(model: &str) -> RoutedRequest {
RoutedRequest {
request: request(),
- decision: Arc::new(TestDecision {
- model: model.to_string(),
- }),
+ decision: test_decision(model.to_string()),
ctx: Context::default(),
}
}
@@ -799,7 +782,7 @@ mod tests {
let Step::CallLlm(call) = step else {
return Err(test_error("expected a CallLlm step"));
};
- calls.insert(call.get_decision().selected_model().to_string(), call);
+ calls.insert(call.get_decision().selected_model_id().to_string(), call);
}
assert!(
tokio::time::timeout(std::time::Duration::from_millis(20), &mut first)
@@ -850,9 +833,7 @@ mod tests {
// A standalone driver reports the typed step receiver disappearing at its next send.
let (driver, step_rx) = Driver::new();
drop(step_rx);
- let decision: Arc<dyn Decision> = Arc::new(TestDecision {
- model: "closed".to_string(),
- });
+ let decision = test_decision("closed".to_string());
let result = driver.info(Context::default(), decision).await;
assert!(matches!(
result,
@@ -877,7 +858,7 @@ mod tests {
/// replaying `chunks` in order (as `Ok` items).
fn streaming_orch(chunks: Vec<LlmResponseChunk>) -> (Arc<dyn Algorithm>, impl Serve) {
let algo = orch(target_set(&["stream/model"]));
- let serve = move |_decision: Arc<dyn Decision>, _request: Request| {
+ let serve = move |_decision: Arc<Decision>, _request: Request| {
let chunks = chunks.clone();
async move {
let stream =
@@ -962,7 +943,7 @@ mod tests {
Step::CallLlm(call) => {
saw_call = true;
// The decision rode along with the promise.
- assert_eq!(call.get_decision().selected_model(), "offload/model");
+ assert_eq!(call.get_decision().selected_model_id(), "offload/model");
// Fulfilling the promise is the "real" model call the caller makes.
call.respond(Ok(Response {
llm_response: LlmResponse::Agg(text_response(
@@ -973,7 +954,7 @@ mod tests {
}))?;
}
Step::Decision(decision) => {
- assert_eq!(decision.selected_model(), "offload/model");
+ assert_eq!(decision.selected_model_id(), "offload/model");
}
Step::ReturnToAgent(response) => {
final_completion = Some(
@@ -1013,7 +994,7 @@ mod tests {
.unwrap_or_default(),
"direct/model"
);
- assert_eq!(trace[0].selected_model(), "direct/model");
+ assert_eq!(trace[0].selected_model_id(), "direct/model");
Ok(())
}
@@ -1036,11 +1017,11 @@ mod tests {
for _ in 0..N {
let algo = algo.clone();
let barrier = barrier.clone();
- let serve = move |decision: Arc<dyn Decision>, _request: Request| {
+ let serve = move |decision: Arc<Decision>, _request: Request| {
let barrier = barrier.clone();
async move {
barrier.wait().await;
- Ok(reply(decision.selected_model()))
+ Ok(reply(decision.selected_model_id()))
}
};
handles.push(tokio::spawn(async move {
@@ -1325,12 +1306,8 @@ mod tests {
driver: Driver,
request: Request,
) -> Result<Response> {
- let dec_w: Arc<dyn Decision> = Arc::new(TestDecision {
- model: self.winner.semantic_name.clone(),
- });
- let dec_l: Arc<dyn Decision> = Arc::new(TestDecision {
- model: self.loser.semantic_name.clone(),
- });
+ let dec_w = test_decision(self.winner.semantic_name.clone());
+ let dec_l = test_decision(self.loser.semantic_name.clone());
let win = driver.call_llm(RoutedRequest {
request: request.clone(),
decision: dec_w,
@@ -1362,10 +1339,10 @@ mod tests {
semantic_name: "loser".to_string(),
},
});
- let serve = move |decision: Arc<dyn Decision>, _request: Request| {
+ let serve = move |decision: Arc<Decision>, _request: Request| {
let started = started.clone();
async move {
- if decision.selected_model() == "loser" {
+ if decision.selected_model_id() == "loser" {
started.notify_one();
match loser_delay {
Some(delay) => tokio::time::sleep(delay).await,
@@ -1374,7 +1351,7 @@ mod tests {
} else {
started.notified().await;
}
- Ok(reply(decision.selected_model()))
+ Ok(reply(decision.selected_model_id()))
}
};
(algo, serve)
@@ -1445,9 +1422,7 @@ mod tests {
request: Request,
) -> Result<Response> {
let offloads = futures::future::join_all((0..self.n).map(|i| {
- let decision: Arc<dyn Decision> = Arc::new(TestDecision {
- model: format!("m{i}"),
- });
+ let decision = test_decision(format!("m{i}"));
driver.call_llm(RoutedRequest {
request: request.clone(),
decision,
@@ -1471,7 +1446,7 @@ mod tests {
// Serving enters each call; once all N are in flight it signals, then pends forever.
let started = Arc::new(AtomicUsize::new(0));
- let serve = move |_decision: Arc<dyn Decision>, _request: Request| {
+ let serve = move |_decision: Arc<Decision>, _request: Request| {
let started = started.clone();
let all_started = all_started.clone();
async move {
diff --git a/crates/libsy/src/core/classifier.rs b/crates/libsy/src/core/classifier.rs
index 7a7451f23..f277b293f 100644
--- a/crates/libsy/src/core/classifier.rs
+++ b/crates/libsy/src/core/classifier.rs
@@ -71,8 +71,8 @@ fn argmax(scores: &[Score]) -> Result<Option<Score>> {
/// Scores targets from the current request and the composition's state.
#[async_trait]
pub trait Classifier<S = ()>: Send + Sync {
- /// Stable tier represented by `selected_model`, when this classifier defines one.
- fn routing_tier(&self, _selected_model: &str) -> Option<&'static str> {
+ /// Stable tier represented by `selected_model_id`, when this classifier defines one.
+ fn routing_tier(&self, _selected_model_id: &str) -> Option<&'static str> {
None
}
diff --git a/crates/libsy/src/core/processor.rs b/crates/libsy/src/core/processor.rs
index 74193cb14..8560402ba 100644
--- a/crates/libsy/src/core/processor.rs
+++ b/crates/libsy/src/core/processor.rs
@@ -18,12 +18,12 @@ pub enum Event<'a> {
/// A routing decision paired with the request that produced it.
///
/// The request is rewritable: a processor may add instructions or notes here that
- /// are bound to the routing outcome (e.g. a tier-specific system prompt).
+ /// are bound to the routing outcome (e.g. a model-specific system prompt).
Decision {
/// The request, rewritable in place.
request: &'a mut Request,
/// The routing decision produced for `request`.
- decision: &'a dyn Decision,
+ decision: &'a Decision,
},
/// A buffered response received back from a model.
ModelResponse(&'a AggLlmResponse),
@@ -71,21 +71,6 @@ mod tests {
}
}
- /// Minimal [`Decision`] so an `Event::Decision` can be constructed.
- struct TestDecision;
-
- impl Decision for TestDecision {
- fn selected_model(&self) -> &str {
- "test/model"
- }
- fn reasoning(&self) -> Option<&str> {
- None
- }
- fn as_any(&self) -> &dyn std::any::Any {
- self
- }
- }
-
fn request() -> Request {
Request {
llm_request: text_request(Some("auto".to_string()), "hi"),
@@ -100,7 +85,7 @@ mod tests {
let mut state = TestState::default();
let mut req = request();
let response = text_response(None, "ok");
- let decision = TestDecision;
+ let decision = Decision::new("test/model", None, true);
let signals = Signals {};
// Feed one of every event variant through the processor.
diff --git a/crates/libsy/src/core/testing.rs b/crates/libsy/src/core/testing.rs
index 755b69474..226e55810 100644
--- a/crates/libsy/src/core/testing.rs
+++ b/crates/libsy/src/core/testing.rs
@@ -29,23 +29,15 @@ pub(crate) type ServeResult = std::result::Result<Response, LlmClientError>;
/// Answers offloaded model calls. Returning `Err` propagates a failed *model* call back into
/// the algorithm, which may route around it.
pub(crate) trait Serve: Send + Sync + 'static {
- fn serve(
- &self,
- decision: Arc<dyn Decision>,
- request: Request,
- ) -> BoxFuture<'static, ServeResult>;
+ fn serve(&self, decision: Arc<Decision>, request: Request) -> BoxFuture<'static, ServeResult>;
}
impl<F, Fut> Serve for F
where
- F: Fn(Arc<dyn Decision>, Request) -> Fut + Send + Sync + 'static,
+ F: Fn(Arc<Decision>, Request) -> Fut + Send + Sync + 'static,
Fut: Future<Output = ServeResult> + Send + 'static,
{
- fn serve(
- &self,
- decision: Arc<dyn Decision>,
- request: Request,
- ) -> BoxFuture<'static, ServeResult> {
+ fn serve(&self, decision: Arc<Decision>, request: Request) -> BoxFuture<'static, ServeResult> {
Box::pin(self(decision, request))
}
}
@@ -56,7 +48,7 @@ pub(crate) async fn test_drive(
ctx: Context,
request: Request,
serve: impl Serve,
-) -> Result<(Vec<Arc<dyn Decision>>, Response)> {
+) -> Result<(Vec<Arc<Decision>>, Response)> {
let serve = Arc::new(serve);
crate::drive(algorithm, ctx, request, move |call| {
fulfill(Arc::clone(&serve), call)
@@ -68,7 +60,7 @@ pub(crate) async fn test_drive(
/// error-shape assertions match production.
async fn fulfill(serve: Arc<impl Serve>, call: CallLlmRequest) -> Result<()> {
let routed = call.get_routed().clone();
- let target = routed.decision.selected_model().to_string();
+ let target = routed.decision.selected_model_id().to_string();
let result = serve
.serve(routed.decision, routed.request)
.await
@@ -79,7 +71,7 @@ async fn fulfill(serve: Arc<impl Serve>, call: CallLlmRequest) -> Result<()> {
/// Answers with the selected model name as the completion — what most routing tests need,
/// since they assert on *which* target was called.
pub(crate) fn echo() -> impl Serve {
- |decision: Arc<dyn Decision>, _request: Request| async move { Ok(reply(decision.selected_model())) }
+ |decision: Arc<Decision>, _request: Request| async move { Ok(reply(decision.selected_model_id())) }
}
/// A buffered response whose completion text is `completion`.
diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs
index fdbcd5e0d..d40328136 100644
--- a/crates/libsy/src/lib.rs
+++ b/crates/libsy/src/lib.rs
@@ -21,9 +21,9 @@ pub use algorithms::llm_class::{
CustomClassifierConfig, CustomClassifierPolicy, LlmClassifierConfig, LlmTaskClassifier,
TaskClassifierConfig,
};
-pub use algorithms::noop::{Noop, NoopDecision};
-pub use algorithms::passthrough::{Passthrough, PassthroughDecision};
-pub use algorithms::rand::{Random, RandomClassifier, RandomDecision};
+pub use algorithms::noop::Noop;
+pub use algorithms::passthrough::Passthrough;
+pub use algorithms::rand::{Random, RandomClassifier};
pub use algorithms::stage::{LlmFallback, StageRouter, StageRouterConfig};
pub use algorithms::util::affinity::AffinityRouter;
pub use algorithms::util::classifier_contract::ClassifierContractConfig;
diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs
index 272a2e5fd..301208114 100644
--- a/crates/libsy/src/observability.rs
+++ b/crates/libsy/src/observability.rs
@@ -211,8 +211,7 @@ pub(crate) fn record_classifier_fail_open(judge_model: &str, reason: &'static st
pub(crate) fn record_llm_call(
algorithm: &str,
selected_model: &str,
- tier: Option<&str>,
- is_routed: bool,
+ is_answer_call: bool,
duration: Duration,
result: &Result<Response>,
span: &Span,
@@ -235,12 +234,9 @@ pub(crate) fn record_llm_call(
.build()
.record(duration.as_secs_f64() * 1000.0, &call_attributes);
- if is_routed {
+ if is_answer_call {
TOTAL_REQUESTS.fetch_add(1, Ordering::Relaxed);
- let mut routed_attributes = vec![KeyValue::new("model", selected_model.to_string())];
- if let Some(tier) = tier {
- routed_attributes.push(KeyValue::new("tier", tier.to_string()));
- }
+ let routed_attributes = [KeyValue::new("model", selected_model.to_string())];
if result.is_ok() {
meter
.u64_counter("switchyard.requests")
@@ -292,9 +288,9 @@ pub(crate) fn record_llm_call(
/// Records one published routing decision: the decision counter plus a
/// structured debug event carrying the decision's reasoning.
-pub(crate) fn record_decision(ctx: &Context, decision: &dyn Decision) {
+pub(crate) fn record_decision(ctx: &Context, decision: &Decision) {
let algorithm = algorithm_label(ctx);
- let selected_model = decision.selected_model();
+ let selected_model = decision.selected_model_id();
tracing::debug!(
target: TRACING_TARGET,
algorithm,
diff --git a/crates/protocol/src/client.rs b/crates/protocol/src/client.rs
index 131ce00dc..5abf5dcb6 100644
--- a/crates/protocol/src/client.rs
+++ b/crates/protocol/src/client.rs
@@ -120,7 +120,7 @@ pub enum RoutingFallbackReason {
}
impl RoutingFallbackReason {
- /// Stable value used by logs and statistics.
+ /// Stable value embedded in routing reasoning.
pub const fn as_str(self) -> &'static str {
match self {
Self::ContextWindow => "context_window",
@@ -129,54 +129,44 @@ impl RoutingFallbackReason {
}
}
-/// A decision/trace object produced by an algorithm.
-///
-/// Carried as a trait object (not a generic parameter) so a stream consumer can
-/// inspect any algorithm's decision through this common interface without
-/// knowing the concrete type. `as_any` is the escape hatch for a consumer that
-/// *does* know the algo and wants to downcast to the concrete decision.
-pub trait Decision: Send + Sync {
- /// The model this decision selected (e.g. the routed target's name).
- fn selected_model(&self) -> &str;
- /// Stable routing tier for the selected model, when the algorithm provides one.
- fn routing_tier(&self) -> Option<&str> {
- None
- }
- /// Whether this is the final call selected to serve the request.
- fn is_routed_call(&self) -> bool {
- true
- }
- /// Why this decision replaced an earlier selected target, when it did.
- fn fallback_reason(&self) -> Option<RoutingFallbackReason> {
- None
- }
- /// A human-readable explanation of the decision, for logs and traces.
- fn reasoning(&self) -> Option<&str>;
- /// Downcast handle: a consumer that knows the algorithm can recover the
- /// concrete decision type via `as_any().downcast_ref::<ConcreteDecision>()`.
- fn as_any(&self) -> &dyn std::any::Any;
+/// A routing choice produced by an algorithm.
+#[derive(Clone, Debug)]
+pub struct Decision {
+ /// The model identifier selected for the call.
+ selected_model_id: String,
+ /// Why, for logs and traces.
+ reasoning: Option<String>,
+ /// True for an answer-generating call. False for classifier and judge calls.
+ is_answer_call: bool,
}
-/// A minimal [`Decision`] implementation for one-off calls that don't belong to a
-/// named algorithm step — judge side calls, classifier side calls, etc.
-pub struct SimpleDecision {
- /// Model or semantic target selected for the call.
- pub selected_model: String,
- /// Optional explanation recorded with the call.
- pub reasoning: Option<String>,
-}
+impl Decision {
+ /// Creates a decision and records whether its call produces the answer.
+ pub fn new(
+ selected_model_id: impl Into<String>,
+ reasoning: Option<String>,
+ is_answer_call: bool,
+ ) -> Self {
+ Self {
+ selected_model_id: selected_model_id.into(),
+ reasoning,
+ is_answer_call,
+ }
+ }
-impl Decision for SimpleDecision {
- fn selected_model(&self) -> &str {
- &self.selected_model
+ /// The model identifier selected for the call.
+ pub fn selected_model_id(&self) -> &str {
+ self.selected_model_id.as_str()
}
- fn reasoning(&self) -> Option<&str> {
+ /// Why this decision was made.
+ pub fn reasoning(&self) -> Option<&str> {
self.reasoning.as_deref()
}
- fn as_any(&self) -> &dyn std::any::Any {
- self
+ /// Whether this call generates an answer rather than a routing verdict.
+ pub fn is_answer_call(&self) -> bool {
+ self.is_answer_call
}
}
@@ -192,15 +182,15 @@ impl Decision for SimpleDecision {
/// not serialize requests unless their transport requires it.
#[async_trait]
pub trait RoutedLlmClient: Send + Sync {
- /// Serve the call, returning the model's response. Call the model named by
- /// [`decision.selected_model()`](Decision::selected_model) — the target the algorithm
- /// routed to — mapping it to whatever provider model id this client hits.
+ /// Serve the model identified by
+ /// [`decision.selected_model_id()`](Decision::selected_model_id), resolving it to the
+ /// provider model this client calls.
/// `request.llm_request.model` is the agent's original name, carried through for
/// reference, not a call target. `ctx` carries the request's cross-cutting state.
async fn call(
&self,
ctx: Context,
request: Request,
- decision: Arc<dyn Decision>,
+ decision: Arc<Decision>,
) -> Result<Response, LlmClientError>;
}
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index d33934706..b4ce79d65 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -35,7 +35,7 @@ impl RoutedLlmClient for PythonLlmClient {
&self,
_ctx: Context,
request: Request,
- _decision: Arc<dyn Decision>,
+ _decision: Arc<Decision>,
) -> Result<Response, LlmClientError> {
let metadata = request.metadata;
let future = Python::attach(|py| {
@@ -268,8 +268,9 @@ impl PyAlgorithm {
.iter()
.map(|decision| {
json!({
- "selected_model": decision.selected_model(),
+ "selected_model_id": decision.selected_model_id(),
"reasoning": decision.reasoning(),
+ "is_answer_call": decision.is_answer_call(),
})
})
.collect::<Vec<Value>>();
diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md
index 221be2196..b5960eb71 100644
--- a/crates/switchyard-server/README.md
+++ b/crates/switchyard-server/README.md
@@ -144,24 +144,21 @@ Routed-call compatibility metrics are:
| `switchyard_build_info` | gauge | `version` | Constant `1` for this server version |
| `switchyard_total_requests` | gauge | none | Successful and failed final routed calls |
| `switchyard_total_errors` | gauge | none | Failed final routed calls |
-| `switchyard_requests_total` | counter | `model`, optional `tier` | Successful final routed calls |
-| `switchyard_errors_total` | counter | `model`, optional `tier` | Failed final routed calls |
-| `switchyard_model_call_latency_ms` | histogram | `model`, optional `tier` | Successful final routed-call latency |
-| `switchyard_prompt_tokens_total` | counter | `model`, optional `tier` | Input tokens, including cached and cache-creation tokens |
-| `switchyard_completion_tokens_total` | counter | `model`, optional `tier` | Output tokens |
-| `switchyard_cached_tokens_total` | counter | `model`, optional `tier` | Cached input tokens |
-| `switchyard_cache_creation_tokens_total` | counter | `model`, optional `tier` | Cache-creation input tokens |
-| `switchyard_reasoning_tokens_total` | counter | `model`, optional `tier` | Reasoning output tokens |
-| `switchyard_total_latency_ms` | histogram | `model`, optional `tier` | Full-turn latency for successful routed responses |
+| `switchyard_requests_total` | counter | `model` | Successful final routed calls |
+| `switchyard_errors_total` | counter | `model` | Failed final routed calls |
+| `switchyard_model_call_latency_ms` | histogram | `model` | Successful final routed-call latency |
+| `switchyard_prompt_tokens_total` | counter | `model` | Input tokens, including cached and cache-creation tokens |
+| `switchyard_completion_tokens_total` | counter | `model` | Output tokens |
+| `switchyard_cached_tokens_total` | counter | `model` | Cached input tokens |
+| `switchyard_cache_creation_tokens_total` | counter | `model` | Cache-creation input tokens |
+| `switchyard_reasoning_tokens_total` | counter | `model` | Reasoning output tokens |
+| `switchyard_total_latency_ms` | histogram | `model` | Full-turn latency for successful routed responses |
| `switchyard_routing_overhead_ms` | histogram | `algorithm` | Algorithm run time minus the call that served it |
| `switchyard_classifier_fail_open_total` | counter | `judge_model`, `reason` | Judge failures that made a classifier route without a verdict |
| `switchyard_client_responses_total` | counter | `outcome` | Final LLM-route responses |
| `switchyard_upstream_attempts_total` | counter | `outcome`, `code` | Actual upstream HTTP attempts |
| `switchyard_router_retry_recovered_total` | counter | none | Retry recoveries (currently always zero) |
-The `tier` label is `strong` or `weak` for a distinguishable built-in LLM-classifier decision and
-is omitted for untiered algorithms. Classifier calls are excluded from these families.
-
`switchyard_classifier_fail_open_total` counts requests that still reached a target after the
judge call failed. `judge_model` names the configured judge target, and `reason` is one of eight
fixed error categories.
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index 2c9414687..fcd9cafdc 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -393,11 +393,10 @@ async fn serve_until_shutdown(
#[derive(Clone, Copy)]
struct RequestStart(Instant);
-/// Tier recorded for classifier and judge calls in the routing log, distinguishing
-/// routing overhead from the routed tiers a session was served by.
+/// Routing-log marker distinguishing classifier and judge calls from answer calls.
const CLASSIFIER_TIER: &str = "classifier";
-/// Maps routed call observations to backend stats, non-routed calls to classifier/judge
+/// Maps answer-call observations to backend stats, routing calls to classifier/judge
/// stats, and records routing overhead once the algorithm run completes.
///
/// Successful classifier/judge calls are also appended to `classifier_log` when one is
@@ -411,11 +410,11 @@ fn stats_observer(
Arc::new(move |observation| match observation {
RunObservation::LlmCall(call) => {
let latency_ms = call.duration.as_secs_f64() * 1_000.0;
- if call.is_routed {
+ if call.is_answer_call {
if call.is_success {
- stats.record_success(&call.selected_model, latency_ms, call.tier.as_deref());
+ stats.record_success(&call.selected_model, latency_ms);
} else {
- stats.record_error(&call.selected_model, call.tier.as_deref());
+ stats.record_error(&call.selected_model);
}
} else if call.is_success {
if let (Some((log, context)), Some(usage)) =
@@ -706,32 +705,22 @@ async fn handle_llm_request(
Ok(result) => result,
Err(error) => return algorithm_error(error),
};
- for reason in trace
- .iter()
- .filter_map(|decision| decision.fallback_reason())
- {
- state.stats.record_routing_fallback(reason);
- }
-
// Metrics, response body, and routing header all read the same decision, so
// the model they name can never disagree. An empty trace leaves the body with
// the id the upstream reported.
let decision = trace.last();
let response = if let Some(decision) = decision {
- let routing_log_context = routing_log_context
- .map(|context| context.with_fallback_reason(decision.fallback_reason()));
let cache_eligible = cache_probe
.as_ref()
.map(|probe| {
state
.stats
- .prefix_eligibility(decision.selected_model(), probe)
+ .prefix_eligibility(decision.selected_model_id(), probe)
})
.unwrap_or(0.0);
usage_metrics::observe(
response,
- decision.selected_model(),
- decision.routing_tier(),
+ decision.selected_model_id(),
started.0,
state.stats,
cache_eligible,
@@ -741,7 +730,7 @@ async fn handle_llm_request(
response
};
- let served_model = decision.map(|decision| decision.selected_model().to_string());
+ let served_model = decision.map(|decision| decision.selected_model_id().to_string());
let mut response = match into_http_response(response, wire_format, served_model) {
Ok(response) => response,
Err(error) => return server_error(error.to_string()),
@@ -823,8 +812,12 @@ fn metadata_from_headers(headers: HeaderMap) -> Metadata {
metadata
}
-fn attach_routing_headers(response: &mut Response, decision: &dyn Decision) {
- insert_routing_header(response, HEADER_SELECTED_MODEL, decision.selected_model());
+fn attach_routing_headers(response: &mut Response, decision: &Decision) {
+ insert_routing_header(
+ response,
+ HEADER_SELECTED_MODEL,
+ decision.selected_model_id(),
+ );
if let Some(reasoning) = decision.reasoning() {
insert_routing_header(response, HEADER_RATIONALE, reasoning);
}
@@ -1335,11 +1328,10 @@ mod tests {
let context = routing_log::RoutingLogContext::from_headers(&headers);
let observer = stats_observer(StatsAccumulator::default(), Some((log.clone(), context)));
- let call = |model: &str, is_routed: bool| {
+ let call = |model: &str, is_answer_call: bool| {
RunObservation::LlmCall(LlmCallObservation {
selected_model: model.to_string(),
- tier: None,
- is_routed,
+ is_answer_call,
is_success: true,
duration: Duration::from_millis(3),
usage: Some(Usage {
diff --git a/crates/switchyard-server/src/routing_log.rs b/crates/switchyard-server/src/routing_log.rs
index c60f47c5e..9060887e5 100644
--- a/crates/switchyard-server/src/routing_log.rs
+++ b/crates/switchyard-server/src/routing_log.rs
@@ -12,7 +12,7 @@ use std::time::SystemTime;
use humantime::format_rfc3339_millis;
use serde::{Deserialize, Serialize};
-use switchyard_protocol::{RoutingFallbackReason, Usage};
+use switchyard_protocol::Usage;
use crate::usage_metrics::token_usage;
use crate::{ServerError, ServerResult};
@@ -56,7 +56,6 @@ impl RoutingLog {
session_id: context.session_id.map(Cow::Owned),
model: model.into(),
tier: tier.unwrap_or("").into(),
- fallback_reason: context.fallback_reason.map(Cow::Borrowed),
prompt_tokens: usage.prompt_tokens,
cached_tokens: usage.cached_tokens,
cache_creation_tokens: usage.cache_creation_tokens,
@@ -102,7 +101,6 @@ pub(crate) struct RoutingLogContext {
task: Option<String>,
trial_id: Option<String>,
session_id: Option<String>,
- fallback_reason: Option<&'static str>,
}
impl RoutingLogContext {
@@ -111,14 +109,8 @@ impl RoutingLogContext {
task: nonempty_header(headers, TASK_HEADER).map(|s| s.to_string()),
trial_id: nonempty_header(headers, TRIAL_ID_HEADER).map(|s| s.to_string()),
session_id: nonempty_header(headers, SESSION_ID_HEADER).map(|s| s.to_string()),
- fallback_reason: None,
}
}
-
- pub(crate) fn with_fallback_reason(mut self, reason: Option<RoutingFallbackReason>) -> Self {
- self.fallback_reason = reason.map(RoutingFallbackReason::as_str);
- self
- }
}
/// One appended routing record, and the read schema [`snapshot`] parses back,
@@ -136,8 +128,6 @@ struct RoutingRecord<'a> {
session_id: Option<Cow<'a, str>>,
model: Cow<'a, str>,
tier: Cow<'a, str>,
- #[serde(borrow, skip_serializing_if = "Option::is_none")]
- fallback_reason: Option<Cow<'a, str>>,
prompt_tokens: u64,
cached_tokens: u64,
cache_creation_tokens: u64,
@@ -244,7 +234,7 @@ mod tests {
fs::write(
&path,
concat!(
- r#"{"session_id":"a","model":"m1","prompt_tokens":10,"completion_tokens":2}"#,
+ r#"{"session_id":"a","model":"m1","fallback_reason":"unavailable","prompt_tokens":10,"completion_tokens":2}"#,
"\n",
r#"{"session_id":"b","model":"m1","prompt_tokens":99,"completion_tokens":99}"#,
"\n",
diff --git a/crates/switchyard-server/src/stats/accumulator.rs b/crates/switchyard-server/src/stats/accumulator.rs
index c7a287991..ee5561808 100644
--- a/crates/switchyard-server/src/stats/accumulator.rs
+++ b/crates/switchyard-server/src/stats/accumulator.rs
@@ -3,12 +3,11 @@
//! Thread-safe stats accumulator and serializable snapshot schema.
-use std::collections::{BTreeMap, BTreeSet, HashSet};
+use std::collections::{BTreeMap, HashSet};
use std::sync::Arc;
use parking_lot::{Mutex, MutexGuard};
use serde::Serialize;
-use switchyard_protocol::RoutingFallbackReason;
use super::cache_eligibility::PrefixProbe;
@@ -33,51 +32,29 @@ pub(crate) struct StatsAccumulator {
impl StatsAccumulator {
/// Records one successful routed backend call.
- pub(crate) fn record_success(
- &self,
- model: impl Into<String>,
- backend_latency_ms: f64,
- tier: Option<&str>,
- ) {
+ pub(crate) fn record_success(&self, model: impl Into<String>, backend_latency_ms: f64) {
let mut inner = self.lock();
inner.total_requests = inner.total_requests.saturating_add(1);
- let model = model.into();
- let tier = normalized_tier(tier);
- let stats = inner.model_stats_mut(model.clone());
+ let stats = inner.model_stats_mut(model.into());
stats.calls = stats.calls.saturating_add(1);
stats.model_call_latency.record(backend_latency_ms);
- if let Some(tier) = tier {
- stats.tiers.insert(tier.to_string());
- let tier_stats = inner.tier_stats_mut(tier, &model);
- tier_stats.calls = tier_stats.calls.saturating_add(1);
- }
}
/// Records one failed routed backend call.
- pub(crate) fn record_error(&self, model: impl Into<String>, tier: Option<&str>) {
+ pub(crate) fn record_error(&self, model: impl Into<String>) {
let mut inner = self.lock();
inner.total_requests = inner.total_requests.saturating_add(1);
inner.total_errors = inner.total_errors.saturating_add(1);
- let model = model.into();
- let stats = inner.model_stats_mut(model.clone());
+ let stats = inner.model_stats_mut(model.into());
stats.errors = stats.errors.saturating_add(1);
- if let Some(tier) = normalized_tier(tier) {
- stats.tiers.insert(tier.to_string());
- inner.tier_stats_mut(tier, &model);
- }
}
/// Records a stream failure after its routed call was already counted.
- pub(crate) fn record_stream_error(&self, model: impl Into<String>, tier: Option<&str>) {
+ pub(crate) fn record_stream_error(&self, model: impl Into<String>) {
let mut inner = self.lock();
inner.total_errors = inner.total_errors.saturating_add(1);
- let model = model.into();
- let stats = inner.model_stats_mut(model.clone());
+ let stats = inner.model_stats_mut(model.into());
stats.errors = stats.errors.saturating_add(1);
- if let Some(tier) = normalized_tier(tier) {
- stats.tiers.insert(tier.to_string());
- inner.tier_stats_mut(tier, &model);
- }
}
/// Records usage and terminal latency after a successful routed call.
@@ -86,17 +63,11 @@ impl StatsAccumulator {
model: impl Into<String>,
usage: TokenUsage,
total_latency_ms: f64,
- tier: Option<&str>,
) {
let mut inner = self.lock();
- let model = model.into();
- let stats = inner.model_stats_mut(model.clone());
+ let stats = inner.model_stats_mut(model.into());
stats.add_usage(usage);
stats.total_latency.record(total_latency_ms);
- if let Some(tier) = normalized_tier(tier) {
- stats.tiers.insert(tier.to_string());
- inner.tier_stats_mut(tier, &model).add_usage(usage);
- }
}
/// Records routing time for one completed algorithm run.
@@ -104,19 +75,6 @@ impl StatsAccumulator {
self.lock().routing_overhead.record(routing_overhead_ms);
}
- /// Records one target replacement by its route-level cause.
- pub(crate) fn record_routing_fallback(&self, reason: RoutingFallbackReason) {
- let fallbacks = &mut self.lock().routing_fallbacks;
- match reason {
- RoutingFallbackReason::ContextWindow => {
- fallbacks.context_window = fallbacks.context_window.saturating_add(1);
- }
- RoutingFallbackReason::Unavailable => {
- fallbacks.unavailable = fallbacks.unavailable.saturating_add(1);
- }
- }
- }
-
/// Records one successful classifier or judge call.
pub(crate) fn record_classifier_success(
&self,
@@ -171,14 +129,9 @@ impl StatsAccumulator {
}
}
-fn normalized_tier(tier: Option<&str>) -> Option<&str> {
- tier.map(str::trim).filter(|tier| !tier.is_empty())
-}
-
#[derive(Clone, Debug, Default)]
struct StatsAccumulatorInner {
by_model: BTreeMap<String, ModelStats>,
- by_tier: BTreeMap<String, TierStats>,
total_requests: u64,
total_errors: u64,
routing_overhead: LatencyHistogram,
@@ -197,12 +150,6 @@ impl StatsAccumulatorInner {
self.by_classifier.entry(model).or_default()
}
- fn tier_stats_mut(&mut self, tier: &str, model: &str) -> &mut TierStats {
- let stats = self.by_tier.entry(tier.to_string()).or_default();
- stats.models.insert(model.to_string());
- stats
- }
-
fn snapshot(&self) -> StatsSnapshot {
let (models, total_tokens) = build_model_snapshots(&self.by_model, self.total_requests);
let classifier = build_classifier_snapshot(
@@ -215,7 +162,6 @@ impl StatsAccumulatorInner {
total_errors: self.total_errors,
total_tokens,
models,
- tiers: tier_snapshots(&self.by_tier, total_tokens.total, self.total_requests),
routing_overhead: self.routing_overhead.snapshot(),
routing_fallbacks: self.routing_fallbacks,
classifier,
@@ -237,7 +183,6 @@ struct ModelStats {
seen_prefixes: HashSet<u64>,
model_call_latency: LatencyHistogram,
total_latency: LatencyHistogram,
- tiers: BTreeSet<String>,
}
impl ModelStats {
@@ -260,23 +205,6 @@ impl ModelStats {
}
}
-#[derive(Clone, Debug, Default)]
-struct TierStats {
- models: BTreeSet<String>,
- calls: u64,
- prompt_tokens: u64,
- completion_tokens: u64,
-}
-
-impl TierStats {
- fn add_usage(&mut self, usage: TokenUsage) {
- self.prompt_tokens = self.prompt_tokens.saturating_add(usage.prompt_tokens);
- self.completion_tokens = self
- .completion_tokens
- .saturating_add(usage.completion_tokens);
- }
-}
-
#[derive(Clone, Debug)]
struct LatencyHistogram {
count: u64,
@@ -344,13 +272,14 @@ pub(crate) struct StatsSnapshot {
pub total_errors: u64,
pub total_tokens: TokenTotals,
pub models: BTreeMap<String, ModelStatsSnapshot>,
- pub tiers: BTreeMap<String, TierStatsSnapshot>,
pub routing_overhead: LatencyHistogramSnapshot,
pub routing_fallbacks: RoutingFallbackStats,
pub classifier: ClassifierStatsSnapshot,
}
-/// Route-level target replacements grouped by their fixed, low-cardinality cause.
+/// Legacy fallback counters retained in the stats response shape.
+///
+/// New decisions carry fallback details in their reasoning instead.
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize)]
pub(crate) struct RoutingFallbackStats {
pub context_window: u64,
@@ -394,18 +323,6 @@ pub(crate) struct ModelStatsSnapshot {
pub theoretical_cache_hit_rate: f64,
pub model_call_latency: LatencyHistogramSnapshot,
pub total_latency: LatencyHistogramSnapshot,
- pub tiers: BTreeSet<String>,
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize)]
-pub(crate) struct TierStatsSnapshot {
- pub models: BTreeSet<String>,
- pub calls: u64,
- pub request_pct: f64,
- pub prompt_tokens: u64,
- pub completion_tokens: u64,
- pub total_tokens: u64,
- pub token_pct: f64,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize)]
@@ -460,7 +377,6 @@ fn build_model_snapshots(
),
model_call_latency: stats.model_call_latency.snapshot(),
total_latency: stats.total_latency.snapshot(),
- tiers: stats.tiers.clone(),
};
(model.clone(), snapshot)
})
@@ -482,31 +398,6 @@ fn build_classifier_snapshot(
}
}
-fn tier_snapshots(
- tiers: &BTreeMap<String, TierStats>,
- total_tokens: u64,
- total_requests: u64,
-) -> BTreeMap<String, TierStatsSnapshot> {
- tiers
- .iter()
- .map(|(tier, stats)| {
- let tier_tokens = stats.prompt_tokens.saturating_add(stats.completion_tokens);
- (
- tier.clone(),
- TierStatsSnapshot {
- models: stats.models.clone(),
- calls: stats.calls,
- request_pct: percentage(stats.calls, total_requests),
- prompt_tokens: stats.prompt_tokens,
- completion_tokens: stats.completion_tokens,
- total_tokens: tier_tokens,
- token_pct: percentage(tier_tokens, total_tokens),
- },
- )
- })
- .collect()
-}
-
fn percentage(numerator: u64, denominator: u64) -> f64 {
if denominator == 0 {
0.0
@@ -557,7 +448,7 @@ mod tests {
#[test]
fn snapshot_aggregates_backend_and_classifier_stats() {
let stats = StatsAccumulator::default();
- stats.record_success("model/strong", 10.0, Some("strong"));
+ stats.record_success("model/strong", 10.0);
stats.record_usage(
"model/strong",
TokenUsage {
@@ -567,11 +458,10 @@ mod tests {
..usage(10, 5)
},
15.0,
- Some("strong"),
);
stats.record_routing_overhead(5.0);
- stats.record_success("model/weak", 20.0, Some("weak"));
- stats.record_usage("model/weak", usage(20, 10), 30.0, Some("weak"));
+ stats.record_success("model/weak", 20.0);
+ stats.record_usage("model/weak", usage(20, 10), 30.0);
stats.record_routing_overhead(10.0);
stats.record_classifier_success("gemini-3.5-flash", Some(usage(1_000_000, 0)), 8.0);
let snapshot = stats.snapshot();
@@ -585,30 +475,17 @@ mod tests {
snapshot.models["model/strong"].theoretical_cache_hit_rate,
0.0
);
- assert_eq!(snapshot.tiers["strong"].request_pct, 50.0);
assert_eq!(snapshot.routing_overhead.p50_ms, 10.0);
assert_eq!(snapshot.routing_overhead.p99_ms, 10.0);
assert_eq!(snapshot.classifier.total_requests, 1);
assert_eq!(snapshot.classifier.total_tokens.prompt, 1_000_000);
-
- stats.record_success("model/strong", 1.0, Some("weak"));
- stats.record_success("model/other", 1.0, Some("strong"));
- let snapshot = stats.snapshot();
- assert_eq!(
- snapshot.models["model/strong"].tiers,
- BTreeSet::from(["strong".to_string(), "weak".to_string()])
- );
- assert_eq!(
- snapshot.tiers["strong"].models,
- BTreeSet::from(["model/other".to_string(), "model/strong".to_string()])
- );
}
#[test]
fn reset_clears_backend_classifier_and_cache_eligibility_state() {
let stats = StatsAccumulator::default();
- stats.record_success("model/a", 10.0, Some("strong"));
- stats.record_usage("model/a", usage(10, 5), 15.0, Some("strong"));
+ stats.record_success("model/a", 10.0);
+ stats.record_usage("model/a", usage(10, 5), 15.0);
stats.record_routing_overhead(5.0);
stats.record_classifier_success("model/classifier", Some(usage(4, 1)), 2.0);
let probe = prefix_probe(&json!({
@@ -636,7 +513,6 @@ mod tests {
..usage(100, 4)
},
1.0,
- None,
);
let second = prefix_probe(&json!({
@@ -653,7 +529,6 @@ mod tests {
..usage(100, 4)
},
1.0,
- None,
);
assert_eq!(first_eligible, 0.0);
diff --git a/crates/switchyard-server/src/usage_metrics.rs b/crates/switchyard-server/src/usage_metrics.rs
index 8632f2dea..1a3e10cba 100644
--- a/crates/switchyard-server/src/usage_metrics.rs
+++ b/crates/switchyard-server/src/usage_metrics.rs
@@ -17,7 +17,6 @@ use crate::stats::{StatsAccumulator, TokenUsage};
pub(crate) fn observe(
response: Response,
model: &str,
- tier: Option<&str>,
started: Instant,
stats: StatsAccumulator,
cache_eligible: f64,
@@ -28,23 +27,12 @@ pub(crate) fn observe(
metadata,
} = response;
let model = model.to_string();
- let tier = tier
- .map(str::trim)
- .filter(|tier| !tier.is_empty())
- .map(str::to_string);
let llm_response = match llm_response {
LlmResponse::Agg(agg) => {
- record_terminal(
- &stats,
- &agg.usage,
- &model,
- tier.as_deref(),
- started,
- cache_eligible,
- );
+ record_terminal(&stats, &agg.usage, &model, started, cache_eligible);
if let Some((log, context)) = routing_log {
- log.append(context, &model, tier.as_deref(), &agg.usage);
+ log.append(context, &model, None, &agg.usage);
}
LlmResponse::Agg(agg)
}
@@ -70,7 +58,7 @@ pub(crate) fn observe(
}
}
if failed {
- record_stream_error(&stats, &model, tier.as_deref());
+ record_stream_error(&stats, &model);
}
yield item;
if failed {
@@ -78,16 +66,9 @@ pub(crate) fn observe(
}
}
let usage = latest_usage.unwrap_or_default();
- record_terminal(
- &stats,
- &usage,
- &model,
- tier.as_deref(),
- started,
- cache_eligible,
- );
+ record_terminal(&stats, &usage, &model, started, cache_eligible);
if let Some((log, context)) = routing_log {
- log.append(context, &model, tier.as_deref(), &usage);
+ log.append(context, &model, None, &usage);
}
};
LlmResponse::Stream(Box::pin(wrapped))
@@ -101,12 +82,12 @@ pub(crate) fn observe(
}
// Records a terminal stream failure after the routed call was already counted.
-fn record_stream_error(stats: &StatsAccumulator, model: &str, tier: Option<&str>) {
- stats.record_stream_error(model, tier);
+fn record_stream_error(stats: &StatsAccumulator, model: &str) {
+ stats.record_stream_error(model);
global::meter("switchyard")
.u64_counter("switchyard.errors")
.build()
- .add(1, &attributes(model, tier));
+ .add(1, &attributes(model));
}
pub(crate) fn token_usage(usage: &Usage) -> TokenUsage {
@@ -131,34 +112,24 @@ fn record_terminal(
stats: &StatsAccumulator,
usage: &Usage,
model: &str,
- tier: Option<&str>,
started: Instant,
cache_eligible: f64,
) {
let total_latency = started.elapsed();
- record_usage(usage, model, tier);
- record_latency(model, tier, total_latency);
+ record_usage(usage, model);
+ record_latency(model, total_latency);
let mut token_usage = token_usage(usage);
token_usage.cacheable_prompt_tokens =
(token_usage.prompt_tokens as f64 * cache_eligible).round() as u64;
- stats.record_usage(
- model,
- token_usage,
- total_latency.as_secs_f64() * 1_000.0,
- tier,
- );
+ stats.record_usage(model, token_usage, total_latency.as_secs_f64() * 1_000.0);
}
-fn attributes(model: &str, tier: Option<&str>) -> Vec<KeyValue> {
- let mut attributes = vec![KeyValue::new("model", model.to_string())];
- if let Some(tier) = tier {
- attributes.push(KeyValue::new("tier", tier.to_string()));
- }
- attributes
+fn attributes(model: &str) -> [KeyValue; 1] {
+ [KeyValue::new("model", model.to_string())]
}
-fn record_usage(usage: &Usage, model: &str, tier: Option<&str>) {
- let attributes = attributes(model, tier);
+fn record_usage(usage: &Usage, model: &str) {
+ let attributes = attributes(model);
let meter = global::meter("switchyard");
let cached = usage.cached_input_tokens();
let cache_creation = usage.cache_creation_input_tokens();
@@ -183,9 +154,9 @@ fn record_usage(usage: &Usage, model: &str, tier: Option<&str>) {
}
}
-fn record_latency(model: &str, tier: Option<&str>, latency: Duration) {
+fn record_latency(model: &str, latency: Duration) {
global::meter("switchyard")
.f64_histogram("switchyard.total_latency_ms")
.build()
- .record(latency.as_secs_f64() * 1000.0, &attributes(model, tier));
+ .record(latency.as_secs_f64() * 1000.0, &attributes(model));
}
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 93f11f986..777830fdd 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -275,7 +275,6 @@ async fn stats_exposes_the_exact_empty_schema_and_no_legacy_alias() -> TestResul
"total_errors": 0,
"total_tokens": empty_token_totals(),
"models": {},
- "tiers": {},
"routing_overhead": {
"count": 0,
"total_ms": 0.0,
@@ -390,7 +389,6 @@ async fn stats_reset_returns_confirmation_and_clears_all_stats() -> TestResult {
assert_eq!(stats["total_errors"], 0);
assert_eq!(stats["total_tokens"], empty_token_totals());
assert_eq!(stats["models"], json!({}));
- assert_eq!(stats["tiers"], json!({}));
assert_eq!(stats["routing_overhead"]["count"], 0);
assert_eq!(stats["classifier"]["total_requests"], 0);
assert_eq!(stats["classifier"]["models"], json!({}));
@@ -905,7 +903,6 @@ base_threshold = 0.5
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(stats["total_requests"], 3);
assert_eq!(stats["models"]["model/weak"]["calls"], 3);
- assert_eq!(stats["tiers"]["weak"]["calls"], 1);
assert_eq!(stats["classifier"]["total_requests"], 1);
assert_eq!(
stats["classifier"]["models"]["model/classifier"]["calls"],
@@ -1672,7 +1669,9 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted
.headers
.get("x-model-router-rationale")
.and_then(|value| value.to_str().ok()),
- Some("model/weak was unavailable; fell back to model/strong")
+ Some(
+ "model/weak was unavailable; fell back to model/strong (fallback reason: unavailable)"
+ )
);
let calls = upstream.calls.lock().await;
assert_eq!(
@@ -1685,7 +1684,8 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted
}
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
- assert_eq!(stats["routing_fallbacks"]["unavailable"], 3);
+ // Fallback cause is now carried in decision reasoning rather than structured stats/logs.
+ assert_eq!(stats["routing_fallbacks"]["unavailable"], 0);
assert_eq!(stats["routing_fallbacks"]["context_window"], 0);
let records = std::fs::read_to_string(&log_path)?;
@@ -1695,7 +1695,7 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted
.collect::<Result<Vec<_>, _>>()?;
assert_eq!(records.len(), 3);
assert!(records.iter().all(|record| {
- record["model"] == "model/strong" && record["fallback_reason"] == "unavailable"
+ record["model"] == "model/strong" && record.get("fallback_reason").is_none()
}));
let previous_call_count = upstream.calls.lock().await.len();
diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py
index b3182cd1c..8067e3ff7 100644
--- a/tests/test_libsy_minimal_bindings.py
+++ b/tests/test_libsy_minimal_bindings.py
@@ -49,8 +49,9 @@ async def test_random_runs_with_a_python_client() -> None:
assert decisions == [
{
- "selected_model": "fast",
+ "selected_model_id": "fast",
"reasoning": "random routing selected target 'fast'",
+ "is_answer_call": True,
}
]
assert client.calls[0]["messages"][0]["content"] == [
@@ -141,7 +142,7 @@ def test_random_rejects_invalid_weights() -> None:
async def test_noop_needs_no_client() -> None:
decisions, response = await algorithms.noop().run(request_body())
- assert decisions[0]["selected_model"] == "auto"
+ assert decisions[0]["selected_model_id"] == "auto"
assert response["outputs"][0]["content"] == [{"type": "text", "text": "OK"}]
@@ -162,7 +163,7 @@ async def test_algorithm_accepts_case_insensitive_duplicate_names() -> None:
request_body(), headers={"X-Unused": "first", "x-unused": "second"}
)
- assert decisions[0]["selected_model"] == "auto"
+ assert decisions[0]["selected_model_id"] == "auto"
def test_algorithm_rejects_header_map_capacity_overflow() -> None: