Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat: Move `Decision::reasoning` to a log message (#413)

Graham Kinga17efa945811
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs
index 9f13e55cb..aefae5bc2 100644
--- a/crates/libsy-llm-client/tests/observability.rs
+++ b/crates/libsy-llm-client/tests/observability.rs
@@ -441,7 +441,8 @@ impl Algorithm for SingleCallAlgo {
.first()
.ok_or(LibsyError::NoTargets)?
.clone();
- let decision = Decision::new(target.clone(), Some(format!("picked '{target}'")), true);
+ tracing::info!("picked '{target}'");
+ let decision = Decision::new(target.clone(), true);
driver.decide(decision.clone()).await?;
driver.call_model(request, decision).await
}
@@ -817,21 +818,15 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l
Some("2")
);
- // Structured debug event: the published decision with its reasoning.
+ // The algorithm logs why it made the decision.
let events = store.events();
assert!(
events.iter().any(|event| {
- event.target == "libsy"
- && event.level == "DEBUG"
- && event.fields.get("selected_model").map(String::as_str) == Some(MODEL)
- && event
- .fields
- .get("reasoning")
- .is_some_and(|reasoning| reasoning.contains("picked"))
+ event.level == "INFO"
&& event
.fields
.get("message")
- .is_some_and(|message| message.contains("routing decision"))
+ .is_some_and(|message| message.contains("picked"))
}),
"no routing-decision log event for {MODEL} in {events:?}"
);
@@ -1229,11 +1224,11 @@ async fn classifier_metrics_count_only_the_final_routed_call() -> switchyard_lib
let (trace, _response) = run(router, client, classifier_request()).await?;
- assert!(
+ assert_eq!(
trace
.last()
- .and_then(|decision| decision.reasoning())
- .is_some_and(|reasoning| reasoning.contains("routing tier: weak"))
+ .map(|decision| decision.selected_model_id().as_str()),
+ Some("weak")
);
let snapshots = flushed_metrics(exporter, provider);
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index 391a24f34..314f3440f 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -142,7 +142,7 @@ where
self
}
- /// Sets the decision reasoning for an algorithm assembled from this cascade.
+ /// Sets the decision log message for an algorithm assembled from this cascade.
pub(crate) fn with_decision_reason(mut self, reason: fn(&str, &Score) -> String) -> Self {
self.decision_reason = reason;
self
@@ -264,17 +264,15 @@ where
RoutingFallbackReason::ContextWindow => "exceeded its context window",
RoutingFallbackReason::Unavailable => "was unavailable",
};
- Decision::new(
- to.clone(),
- Some(with_routing_tier(
- format!(
- "{from} {failure}; fell back to {to} (fallback reason: {})",
- reason.as_str(),
- ),
- deciding.routing_tier(to),
- )),
- true,
- )
+ let message = with_routing_tier(
+ format!(
+ "{from} {failure}; fell back to {to} (fallback reason: {})",
+ reason.as_str(),
+ ),
+ deciding.routing_tier(to),
+ );
+ tracing::info!("{message}");
+ Decision::new(to.clone(), true)
}
/// Returns this request's retained state without holding the registry lock.
@@ -321,10 +319,10 @@ where
});
};
- // 3. Resolve the target and publish the decision. When an excluded target sends
- // the request elsewhere, the reasoning describes where it actually went.
+ // 3. Resolve the target, log the choice, and publish the decision. When an excluded
+ // target sends the request elsewhere, the log describes where it actually went.
let target = algorithm::select_eligible_model(&self.targets, &score.target, excluded)?;
- let reasoning = if target == score.target {
+ let message = if target == score.target {
(self.decision_reason)(&self.name, &score)
} else {
format!(
@@ -332,11 +330,9 @@ where
score.target, target
)
};
- let decision: Decision = Decision::new(
- target.clone(),
- Some(with_routing_tier(reasoning, deciding.routing_tier(&target))),
- true,
- );
+ let message = with_routing_tier(message, deciding.routing_tier(&target));
+ tracing::info!("{message}");
+ let decision: Decision = Decision::new(target.clone(), true);
driver.decide(decision.clone()).await?;
// 4. Post-decision replay: every processor sees the decision so stateful ones
@@ -395,11 +391,11 @@ fn default_decision_reason(_name: &str, winner: &Score) -> String {
)
}
-// Routing tier is not part of Decision struct, so keeping it in reasoning for now. Remove it later if needed from reasoning.
-fn with_routing_tier(reasoning: String, tier: Option<&str>) -> String {
+/// Appends the routing tier to a decision log message when the classifier supplies one.
+fn with_routing_tier(message: String, tier: Option<&str>) -> String {
match tier {
- Some(tier) => format!("{reasoning}; routing tier: {tier}"),
- None => reasoning,
+ Some(tier) => format!("{message}; routing tier: {tier}"),
+ None => message,
}
}
@@ -836,11 +832,11 @@ mod tests {
for _ in 0..2 {
let (model, trace) = run_turn(&router, unavailable(&["weak"], calls.clone())).await?;
assert_eq!(model, "strong");
- assert!(
+ assert_eq!(
trace
.last()
- .and_then(|decision| decision.reasoning())
- .is_some_and(|reasoning| reasoning.contains("fallback reason: unavailable"))
+ .map(|decision| decision.selected_model_id().as_str()),
+ Some("strong")
);
}
assert_eq!(&*calls.lock(), &["weak", "strong", "weak", "strong"]);
@@ -848,22 +844,11 @@ mod tests {
}
#[tokio::test]
- async fn fallback_decision_preserves_tier_and_cause_in_reasoning() -> Result<()> {
+ async fn fallback_decision_preserves_answer_call_semantics() -> Result<()> {
struct TieredClassifier;
#[async_trait]
impl Classifier for TieredClassifier {
- fn routing_tier(
- &self,
- selected_model_id: &switchyard_protocol::ModelId,
- ) -> Option<&'static str> {
- match selected_model_id.as_str() {
- "weak" => Some("weak"),
- "strong" => Some("strong"),
- _ => None,
- }
- }
-
async fn score(
&self,
_state: &mut (),
@@ -881,19 +866,10 @@ mod tests {
assert_eq!(trace.len(), 2);
assert_eq!(trace[0].selected_model_id(), "weak");
assert!(trace[0].is_answer_call());
- assert!(
- trace[0]
- .reasoning()
- .is_some_and(|reasoning| reasoning.contains("routing tier: weak"))
- );
let fallback = &trace[1];
assert_eq!(fallback.selected_model_id(), "strong");
assert!(fallback.is_answer_call());
- assert!(fallback.reasoning().is_some_and(|reasoning| {
- reasoning.contains("fallback reason: unavailable")
- && reasoning.contains("routing tier: strong")
- }));
Ok(())
}
@@ -964,11 +940,6 @@ mod tests {
assert_eq!(text, "strong");
assert_eq!(trace[0].selected_model_id(), "strong");
- assert!(
- trace[0]
- .reasoning()
- .is_some_and(|r| r.contains("fell back to strong"))
- );
Ok(())
}
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index b60ebea63..f00b31ccb 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -513,15 +513,12 @@ impl Classifier<State> for EscalationClassifier {
// If the efficient model exceeds its context window, fall through to capable: returning
// `(decisive(capable), None)` tells FallThrough::execute to call
// call_model_with_fallback with the capable target instead of surfacing the error.
+ tracing::info!(
+ target = %self.efficient,
+ "escalation classifier selected efficient tier"
+ );
let efficient_response = match driver
- .call_model(
- request.clone(),
- Decision::new(
- self.efficient.clone(),
- Some("escalation classifier: efficient tier".into()),
- true,
- ),
- )
+ .call_model(request.clone(), Decision::new(self.efficient.clone(), true))
.await
{
Ok(r) => r,
@@ -1822,12 +1819,6 @@ mod tests {
trace.last().map(|d| d.selected_model_id().as_str()),
Some("efficient")
);
- assert!(
- trace
- .last()
- .and_then(|decision| decision.reasoning())
- .is_some_and(|reasoning| reasoning.contains("routing tier: weak"))
- );
assert_eq!(
response.llm_response.as_agg().map(completion_text),
Some("efficient answer".to_string())
@@ -1873,12 +1864,6 @@ mod tests {
trace.last().map(|d| d.selected_model_id().as_str()),
Some("capable")
);
- assert!(
- trace
- .last()
- .and_then(|decision| decision.reasoning())
- .is_some_and(|reasoning| reasoning.contains("routing tier: strong"))
- );
assert_eq!(
response.llm_response.as_agg().map(completion_text),
Some("capable answer".to_string())
diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rs
index acf240972..90e053b7a 100644
--- a/crates/libsy/src/algorithms/noop.rs
+++ b/crates/libsy/src/algorithms/noop.rs
@@ -27,11 +27,8 @@ impl Algorithm for Noop {
let model_id = request
.model_id()
.unwrap_or_else(|| ModelId::from("switchyard/noop"));
- let decision: Decision = Decision::new(
- model_id.clone(),
- Some("noop returned its synthetic response".to_string()),
- true,
- );
+ tracing::info!(target = %model_id, "noop returned its synthetic response");
+ let decision: Decision = Decision::new(model_id.clone(), true);
driver.decide(decision.clone()).await?;
let llm_response = LlmResponse::Agg(AggLlmResponse {
diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rs
index 0727175b3..f123c1d4f 100644
--- a/crates/libsy/src/algorithms/passthrough.rs
+++ b/crates/libsy/src/algorithms/passthrough.rs
@@ -32,11 +32,8 @@ impl Algorithm for Passthrough {
}
async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {
- let decision: Decision = Decision::new(
- self.target.clone(),
- Some(format!("passthrough selected target '{}'", self.target)),
- true,
- );
+ tracing::info!(target = %self.target, "passthrough selected target");
+ let decision: Decision = Decision::new(self.target.clone(), true);
driver.decide(decision.clone()).await?;
driver.call_model(request, decision).await
}
diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rs
index bb52aa331..4be406f32 100644
--- a/crates/libsy/src/algorithms/rand.rs
+++ b/crates/libsy/src/algorithms/rand.rs
@@ -397,12 +397,6 @@ mod tests {
let decision = &trace[0];
assert_eq!(decision.selected_model_id(), "only/model");
- assert!(
- decision
- .reasoning()
- .unwrap_or_default()
- .contains("only/model")
- );
assert!(decision.is_answer_call());
Ok(())
}
diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs
index 6512d7636..948793586 100644
--- a/crates/libsy/src/algorithms/stage.rs
+++ b/crates/libsy/src/algorithms/stage.rs
@@ -511,11 +511,11 @@ mod tests {
"the selected target should be recorded as an answer call"
);
drop(calls);
- assert!(
+ assert_eq!(
trace
.last()
- .and_then(|decision| decision.reasoning())
- .is_some_and(|reasoning| reasoning.contains("routing tier: strong"))
+ .map(|decision| decision.selected_model_id().as_str()),
+ Some("strong")
);
Ok(())
}
diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rs
index 5ffc869cc..d372fdef0 100644
--- a/crates/libsy/src/algorithms/util/affinity.rs
+++ b/crates/libsy/src/algorithms/util/affinity.rs
@@ -238,7 +238,7 @@ mod tests {
type BoxErr = Box<dyn std::error::Error + Send + Sync>;
fn fixed_decision(target: &str) -> Decision {
- Decision::new(target, None, true)
+ Decision::new(target, true)
}
fn request(metadata: Metadata) -> Request {
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
index bb271ac5a..63223bd6c 100644
--- a/crates/libsy/src/algorithms/util/llm_judge.rs
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -226,14 +226,11 @@ where
) -> Option<J::Verdict> {
let judge_model = self.target.as_str();
+ tracing::info!(target = judge_model, "consulting llm judge");
let response = driver
.call_model(
self.judge.build_request(state, request),
- Decision::new(
- self.target.to_string(),
- Some("llm judge consultation".to_string()),
- false,
- ),
+ Decision::new(self.target.to_string(), false),
)
.await
.inspect_err(|error| report_fail_open(judge_model, error, libsy_error_reason(error)))
diff --git a/crates/libsy/src/algorithms/util/prompts.rs b/crates/libsy/src/algorithms/util/prompts.rs
index e6fd6c673..8290c6346 100644
--- a/crates/libsy/src/algorithms/util/prompts.rs
+++ b/crates/libsy/src/algorithms/util/prompts.rs
@@ -265,7 +265,7 @@ mod tests {
},
..Request::default()
};
- let decision = Decision::new(target, None, true);
+ let decision = Decision::new(target, true);
processor
.process(
&mut (),
@@ -326,7 +326,7 @@ mod tests {
text: "you are a coding agent".to_string(),
}],
});
- let decision = Decision::new("strong", None, true);
+ let decision = Decision::new("strong", true);
processor
.process(
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index 129474863..d75163f8c 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -192,8 +192,8 @@ impl Driver {
}
/// Publish a routing [`Decision`] as a [`Step::Decision`] on the stream.
- /// Each successfully published decision is counted and logged with its
- /// reasoning; a decision the stream never accepted is not recorded.
+ /// Each successfully published decision is counted and logged; a decision
+ /// the stream never accepted is not recorded.
pub async fn decide(&self, decision: Decision) -> Result<()> {
self.step_tx
.send(Ok(Step::Decision(decision.clone())))
@@ -660,7 +660,7 @@ mod tests {
/// Build a routed decision for orchestration tests.
fn test_decision(selected_model_id: ModelId) -> Decision {
- Decision::new(selected_model_id, None, true)
+ Decision::new(selected_model_id, true)
}
/// Trivial algo used only to exercise the orchestrator: calls the first target
diff --git a/crates/libsy/src/core/processor.rs b/crates/libsy/src/core/processor.rs
index b84727bb0..d6047e259 100644
--- a/crates/libsy/src/core/processor.rs
+++ b/crates/libsy/src/core/processor.rs
@@ -82,7 +82,7 @@ mod tests {
let mut state = TestState::default();
let mut req = request();
let response = text_response(None, "ok");
- let decision = Decision::new("test/model", None, true);
+ let decision = Decision::new("test/model", true);
// Feed one of every event variant through the processor.
processor
.process(&mut state, Event::Request(&mut req))
diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs
index 36d39c4e1..e0f1f06c2 100644
--- a/crates/libsy/src/observability.rs
+++ b/crates/libsy/src/observability.rs
@@ -285,15 +285,13 @@ pub(crate) fn record_llm_call(
}
}
-/// Records one published routing decision: the decision counter plus a
-/// structured debug event carrying the decision's reasoning.
+/// Records one published routing decision: the decision counter plus a structured debug event.
pub(crate) fn record_decision(algorithm: &str, decision: &Decision) {
let selected_model = decision.selected_model_id();
tracing::debug!(
target: TRACING_TARGET,
algorithm,
selected_model = %selected_model,
- reasoning = decision.reasoning().unwrap_or(""),
"routing decision"
);
meter().u64_counter("switchyard.decisions").build().add(
diff --git a/crates/protocol/src/client.rs b/crates/protocol/src/client.rs
index 3b7585039..f98155f16 100644
--- a/crates/protocol/src/client.rs
+++ b/crates/protocol/src/client.rs
@@ -118,7 +118,7 @@ pub enum RoutingFallbackReason {
}
impl RoutingFallbackReason {
- /// Stable value embedded in routing reasoning.
+ /// Stable value used when logging a routing fallback.
pub const fn as_str(self) -> &'static str {
match self {
Self::ContextWindow => "context_window",
@@ -132,22 +132,15 @@ impl RoutingFallbackReason {
pub struct Decision {
/// The model identifier selected for the call.
selected_model_id: ModelId,
- /// Why, for logs and traces.
- reasoning: Option<String>,
/// True for an answer-generating call. False for classifier and judge calls.
is_answer_call: bool,
}
impl Decision {
/// Creates a decision and records whether its call produces the answer.
- pub fn new(
- selected_model_id: impl Into<ModelId>,
- reasoning: Option<String>,
- is_answer_call: bool,
- ) -> Self {
+ pub fn new(selected_model_id: impl Into<ModelId>, is_answer_call: bool) -> Self {
Self {
selected_model_id: selected_model_id.into(),
- reasoning,
is_answer_call,
}
}
@@ -157,11 +150,6 @@ impl Decision {
&self.selected_model_id
}
- /// Why this decision was made.
- pub fn reasoning(&self) -> Option<&str> {
- self.reasoning.as_deref()
- }
-
/// Whether this call generates an answer rather than a routing verdict.
pub fn is_answer_call(&self) -> bool {
self.is_answer_call
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index ce303d5e6..3b3423bb9 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -151,12 +151,6 @@ impl PyDecision {
self.inner.selected_model_id().as_str()
}
- /// Why the algorithm selected this model, when supplied.
- #[getter]
- fn reasoning(&self) -> Option<&str> {
- self.inner.reasoning()
- }
-
/// Whether this call produces the answer rather than a routing verdict.
#[getter]
fn is_answer_call(&self) -> bool {
@@ -165,9 +159,8 @@ impl PyDecision {
fn __repr__(&self) -> String {
format!(
- "Decision(selected_model_id={:?}, reasoning={:?}, is_answer_call={})",
+ "Decision(selected_model_id={:?}, is_answer_call={})",
self.inner.selected_model_id(),
- self.inner.reasoning(),
self.inner.is_answer_call()
)
}
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index 9c538ea0d..81ee7d6ae 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -59,7 +59,6 @@ pub const DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30);
pub const DEFAULT_MAX_REQUEST_BODY_BYTES: usize = 32 * 1024 * 1024;
const HEADER_SELECTED_MODEL: &str = "x-model-router-selected-model";
-const HEADER_RATIONALE: &str = "x-model-router-rationale";
const MAX_ROUTING_HEADER_VALUE_LEN: usize = 512;
const STARTUP_BANNER_ART: &str = include_str!("../assets/startup_banner.txt");
@@ -815,9 +814,6 @@ fn attach_routing_headers(response: &mut Response, decision: &Decision) {
HEADER_SELECTED_MODEL,
decision.selected_model_id(),
);
- if let Some(reasoning) = decision.reasoning() {
- insert_routing_header(response, HEADER_RATIONALE, reasoning);
- }
}
fn insert_routing_header(response: &mut Response, name: &'static str, value: &str) {
diff --git a/crates/switchyard-server/src/stats/accumulator.rs b/crates/switchyard-server/src/stats/accumulator.rs
index 7ddfb06e5..281609229 100644
--- a/crates/switchyard-server/src/stats/accumulator.rs
+++ b/crates/switchyard-server/src/stats/accumulator.rs
@@ -325,7 +325,7 @@ pub(crate) struct StatsSnapshot {
/// Legacy fallback counters retained in the stats response shape.
///
-/// New decisions carry fallback details in their reasoning instead.
+/// New fallback details are logged instead.
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize)]
pub(crate) struct RoutingFallbackStats {
pub context_window: u64,
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index a1241c59d..10f58b65c 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -1664,15 +1664,6 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted
.and_then(|value| value.to_str().ok()),
Some("model/strong")
);
- assert_eq!(
- response
- .headers
- .get("x-model-router-rationale")
- .and_then(|value| value.to_str().ok()),
- Some(
- "model/weak was unavailable; fell back to model/strong (fallback reason: unavailable)"
- )
- );
let calls = upstream.calls.lock().await;
assert_eq!(
calls[previous_call_count..]
@@ -1684,7 +1675,7 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted
}
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
- // Fallback cause is now carried in decision reasoning rather than structured stats/logs.
+ // Fallback causes are logged rather than accumulated in the legacy stats counters.
assert_eq!(stats["routing_fallbacks"]["unavailable"], 0);
assert_eq!(stats["routing_fallbacks"]["context_window"], 0);
diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py
index 988454635..00ca23b1c 100644
--- a/tests/test_libsy_minimal_bindings.py
+++ b/tests/test_libsy_minimal_bindings.py
@@ -100,7 +100,6 @@ async def test_random_streams_complex_steps_and_accepts_a_dictionary_response()
assert variants == ["decision", "call_model", "done"]
assert len(decisions) == 1
assert decisions[0].selected_model_id == "fast"
- assert decisions[0].reasoning == "random routing selected target 'fast'"
assert decisions[0].is_answer_call is True
assert client.calls[0]["model"] == "fast"
assert client.calls[0]["messages"][0]["content"] == [