Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(runner): let targets share one model with different reasoning effort (#991)

sabhatinasf15599493a47
diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rs
index b24c1a0d7..7e41b502f 100644
--- a/crates/libsy-llm-client/src/backend.rs
+++ b/crates/libsy-llm-client/src/backend.rs
@@ -76,8 +76,8 @@ pub struct HttpBackendConfig {
/// standard field (e.g. an internal API that 400s on `max_output_tokens`).
pub omit_body_fields: BTreeSet<String>,
/// Reasoning effort forced on every request to this backend, replacing whatever the caller
- /// sent. Responses carries it as `reasoning.effort`, Chat Completions as `reasoning_effort`;
- /// Anthropic has no equivalent and rejects the setting at configuration time.
+ /// sent. Responses carries it as `reasoning.effort`, Chat Completions as `reasoning_effort`,
+ /// and Anthropic Messages as `output_config.effort`.
pub reasoning_effort: Option<String>,
/// Additional attempts for transient upstream failures.
/// One template-compatibility retry may also run within the same timeout.
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index deb4d9a12..b17652a15 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -64,6 +64,7 @@ const MAX_RETRY_AFTER: Duration = Duration::from_secs(60);
#[derive(Clone, Debug)]
pub struct ModelConfig {
model_name: ModelId,
+ provider_model: Option<ModelId>,
default_backend: Backend,
other_backends: Option<Vec<Backend>>,
responses_reasoning: crate::ResponsesReasoningPolicy,
@@ -79,6 +80,7 @@ impl ModelConfig {
) -> Self {
Self {
model_name: model_name.into(),
+ provider_model: None,
default_backend,
other_backends,
responses_reasoning: crate::ResponsesReasoningPolicy::default(),
@@ -91,6 +93,14 @@ impl ModelConfig {
self.responses_reasoning = policy;
self
}
+
+ /// Sends `provider_model` upstream in place of `model_name`. The client is still
+ /// called with `model_name`, so several entries can serve one provider model with
+ /// different backend settings.
+ pub fn with_provider_model(mut self, provider_model: impl Into<ModelId>) -> Self {
+ self.provider_model = Some(provider_model.into());
+ self
+ }
}
/// A model-bearing provider operation outside the normal completion endpoint.
@@ -205,6 +215,15 @@ impl TranslatingLlmClient {
})
}
+ /// The model id sent upstream for `model`: its provider model when one is set,
+ /// otherwise `model` itself.
+ pub fn provider_model<'a>(&'a self, model: &'a ModelId) -> &'a ModelId {
+ self.model_to_config
+ .get(model)
+ .and_then(|config| config.provider_model.as_ref())
+ .unwrap_or(model)
+ }
+
/// Whether `model` has a backend for `operation`.
pub fn supports_auxiliary(&self, model: &ModelId, operation: AuxiliaryOperation) -> bool {
self.backend_for(model, operation.wire_format()).is_some()
@@ -281,7 +300,7 @@ impl TranslatingLlmClient {
metadata,
..
} = request;
- llm_request.model = Some(model.to_string());
+ llm_request.model = Some(self.provider_model(model).to_string());
let http_response = self
.send_encoded(
backend,
@@ -340,8 +359,9 @@ impl TranslatingLlmClient {
.map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?;
// `encode_request` round-trips a preserved same-format body verbatim,
// which keeps the caller's original `model`; force the resolved model so
- // the upstream always sees the target id.
- set_json_model(&mut body, model);
+ // the upstream always sees the provider model.
+ let upstream_model = self.provider_model(model);
+ set_json_model(&mut body, upstream_model);
if matches!(backend, Backend::OpenAiResponses(_)) {
sanitize_openai_responses_provider_body(&mut body);
}
@@ -361,8 +381,11 @@ impl TranslatingLlmClient {
.normalize(&mut body);
}
// After the merge on purpose: the effort override must win over both the caller's
- // value and any `reasoning` default a target set through `extra_body`.
- apply_reasoning_effort(&mut body, backend);
+ // value and any `reasoning` default a target set through `extra_body`. Token counting
+ // does not depend on effort, and Anthropic's count endpoint may reject the field.
+ if matches!(endpoint, UpstreamEndpoint::Completion) {
+ apply_reasoning_effort(&mut body, backend);
+ }
if matches!(backend, Backend::OpenAiResponses(_)) {
normalize_codex_string_input(&backend.url(), &mut body);
}
@@ -378,7 +401,7 @@ impl TranslatingLlmClient {
let streaming = endpoint.allows_streaming()
&& body.get("stream").and_then(Value::as_bool).unwrap_or(false);
let url = endpoint.url(backend);
- record_gen_ai_request(&url, model, streaming);
+ record_gen_ai_request(&url, upstream_model, streaming);
let result = self
.send_with_retries(&url, backend, &body, metadata, model, streaming)
@@ -645,8 +668,8 @@ impl TranslatingLlmClient {
/// backend), and returns the neutral response.
///
/// Resolution: `model_name` wins over `request.llm_request.model`; the
- /// resolved name is both the outer map key and the model id written into the
- /// request before translation. Missing models are invalid requests; unknown
+ /// resolved name is the outer map key, and its provider model is the model id
+ /// written into the request before translation. Missing models are invalid requests; unknown
/// models or wire formats are configuration errors.
pub async fn call_rewrite_model(
&self,
@@ -665,7 +688,7 @@ impl TranslatingLlmClient {
.ok_or_else(|| LlmClientError::InvalidRequest {
message: "no model given".to_string(),
})?;
- llm_request.model = Some(model_id.to_string());
+ llm_request.model = Some(self.provider_model(&model_id).to_string());
let orig_format = metadata.as_ref().and_then(|m| m.wire_format);
let wire_format = orig_format.unwrap_or(
@@ -775,8 +798,9 @@ impl TranslatingLlmClient {
// one, else the request's own model. Mirrors `call_rewrite_model`'s own
// resolution so the response names whoever answered.
let served_model = model
- .map(ModelId::to_string)
- .or_else(|| llm_request.model.clone());
+ .cloned()
+ .or_else(|| llm_request.model.clone().map(ModelId::from))
+ .map(|model| self.provider_model(&model).to_string());
let request = Request {
llm_request,
@@ -820,6 +844,10 @@ impl TranslatingLlmClient {
#[async_trait]
impl RoutedLlmClient for TranslatingLlmClient {
+ fn provider_model<'a>(&'a self, target: &'a ModelId) -> &'a ModelId {
+ TranslatingLlmClient::provider_model(self, target)
+ }
+
async fn call(&self, request: Request) -> Result<Response> {
self.call_rewrite_model(request, None).await
}
@@ -1266,6 +1294,16 @@ fn strip_anthropic_incompatible_fields(body: &mut Value) {
if let Value::Object(object) = body {
object.remove("reasoning_effort");
object.remove("context_management");
+ // `strict: false` is the default, and Bedrock rejects the field outright.
+ if let Some(Value::Array(tools)) = object.get_mut("tools") {
+ for tool in tools {
+ if let Value::Object(tool) = tool
+ && tool.get("strict") == Some(&Value::Bool(false))
+ {
+ tool.remove("strict");
+ }
+ }
+ }
}
}
@@ -1351,20 +1389,9 @@ fn apply_reasoning_effort(body: &mut Value, backend: &Backend) {
return;
};
match backend {
- Backend::OpenAiResponses(_) => {
- // Responses nests effort under `reasoning` next to fields the caller may have set
- // (`summary`, for example), so only the `effort` key is replaced. A `reasoning`
- // value that is not an object is malformed and is replaced whole.
- let reasoning = object
- .entry("reasoning".to_string())
- .or_insert_with(|| Value::Object(serde_json::Map::new()));
- if !reasoning.is_object() {
- *reasoning = Value::Object(serde_json::Map::new());
- }
- if let Value::Object(reasoning) = reasoning {
- reasoning.insert("effort".to_string(), Value::String(effort.to_string()));
- }
- }
+ // Responses nests effort under `reasoning`, Anthropic under `output_config`.
+ Backend::OpenAiResponses(_) => set_nested_effort(object, "reasoning", effort),
+ Backend::Anthropic(_) => set_nested_effort(object, "output_config", effort),
Backend::OpenAiChat(_) => {
// Chat Completions takes effort as a top-level field.
object.insert(
@@ -1372,9 +1399,20 @@ fn apply_reasoning_effort(body: &mut Value, backend: &Backend) {
Value::String(effort.to_string()),
);
}
- // Anthropic has no effort field (thinking is a token budget); the runner rejects the
- // setting on Anthropic clients at load time, so this arm is unreachable in practice.
- Backend::Anthropic(_) => {}
+ }
+}
+
+// Replaces only `effort` inside `key`, keeping fields the caller set next to it (`summary` or
+// `format`, for example). A non-object value there is malformed and is replaced whole.
+fn set_nested_effort(object: &mut serde_json::Map<String, Value>, key: &str, effort: &str) {
+ let container = object
+ .entry(key.to_string())
+ .or_insert_with(|| Value::Object(serde_json::Map::new()));
+ if !container.is_object() {
+ *container = Value::Object(serde_json::Map::new());
+ }
+ if let Value::Object(container) = container {
+ container.insert("effort".to_string(), Value::String(effort.to_string()));
}
}
@@ -2332,6 +2370,69 @@ mod tests {
Ok(())
}
+ // Covers both a translated request and an Anthropic caller's body sent as is.
+ #[tokio::test]
+ async fn anthropic_requests_drop_tool_strict_false()
+ -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/v1/messages"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "id": "msg_1",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude",
+ "content": [{"type": "text", "text": "ok"}],
+ "stop_reason": "end_turn",
+ "usage": {"input_tokens": 1, "output_tokens": 1}
+ })))
+ .expect(2)
+ .mount(&server)
+ .await;
+ let client = TranslatingLlmClient::new(&[ModelConfig::new(
+ "claude",
+ Backend::Anthropic(config(&server.uri())),
+ None,
+ )])?;
+ let schema = json!({"type": "object"});
+ let requests = [
+ (
+ WireFormat::OpenAiChat,
+ json!({
+ "model": "client-facing",
+ "messages": [{"role": "user", "content": "hi"}],
+ "tools": [
+ {"type": "function", "function": {"name": "loose", "parameters": schema, "strict": false}},
+ {"type": "function", "function": {"name": "exact", "parameters": schema, "strict": true}}
+ ]
+ }),
+ ),
+ (
+ WireFormat::AnthropicMessages,
+ json!({
+ "model": "client-facing",
+ "max_tokens": 8,
+ "messages": [{"role": "user", "content": "hi"}],
+ "tools": [
+ {"name": "loose", "input_schema": schema, "strict": false},
+ {"name": "exact", "input_schema": schema, "strict": true}
+ ]
+ }),
+ ),
+ ];
+ for (format, request) in requests {
+ client
+ .call_rewrite_model_raw(request, None, Some(&ModelId::from("claude")), format)
+ .await?;
+ }
+ for received in server.received_requests().await.unwrap_or_default() {
+ let body: Value = serde_json::from_slice(&received.body)?;
+ assert!(body["tools"][0].get("strict").is_none(), "{body}");
+ assert_eq!(body["tools"][1]["strict"], true, "{body}");
+ }
+ Ok(())
+ }
+
#[tokio::test]
async fn template_rejection_retries_once_without_changing_other_fields()
-> std::result::Result<(), Box<dyn Error + Sync + Send>> {
@@ -2491,6 +2592,91 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn token_counting_does_not_send_the_effort_override()
+ -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/v1/messages/count_tokens"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({"input_tokens": 3})))
+ .expect(1)
+ .mount(&server)
+ .await;
+ let mut backend = config(&server.uri());
+ backend.reasoning_effort = Some("max".to_string());
+ let client = TranslatingLlmClient::new(&[ModelConfig::new(
+ "claude",
+ Backend::Anthropic(backend),
+ None,
+ )])?;
+ let llm_request = switchyard_translation::decode_request(
+ WireFormat::AnthropicMessages,
+ &json!({"model": "claude", "messages": [{"role": "user", "content": "hi"}]}),
+ )?;
+ let request = Request {
+ llm_request,
+ raw_request: None,
+ metadata: None,
+ };
+ client
+ .call_auxiliary(
+ &ModelId::from("claude"),
+ request,
+ AuxiliaryOperation::AnthropicCountTokens,
+ )
+ .await?;
+ let received = server.received_requests().await.unwrap_or_default();
+ let body: Value = serde_json::from_slice(&received[0].body)?;
+ assert!(body.get("output_config").is_none(), "{body}");
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn reasoning_effort_override_sets_anthropic_output_config_effort()
+ -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ let format = json!({"type": "json_schema", "schema": {"type": "object"}});
+ Mock::given(method("POST"))
+ .and(path("/v1/messages"))
+ .and(wiremock::matchers::body_partial_json(json!({
+ "model": "claude",
+ "output_config": {"effort": "max", "format": format}
+ })))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "id": "msg_1",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude",
+ "content": [{"type": "text", "text": "ok"}],
+ "stop_reason": "end_turn",
+ "usage": {"input_tokens": 1, "output_tokens": 1}
+ })))
+ .expect(1)
+ .mount(&server)
+ .await;
+ let mut backend = config(&server.uri());
+ backend.reasoning_effort = Some("max".to_string());
+ let client = TranslatingLlmClient::new(&[ModelConfig::new(
+ "claude",
+ Backend::Anthropic(backend),
+ None,
+ )])?;
+ client
+ .call_rewrite_model_raw(
+ json!({
+ "model": "client-facing",
+ "max_tokens": 7,
+ "messages": [{"role": "user", "content": "hi"}],
+ "output_config": {"effort": "high", "format": format}
+ }),
+ None,
+ Some(&ModelId::from("claude")),
+ WireFormat::AnthropicMessages,
+ )
+ .await?;
+ Ok(())
+ }
+
#[tokio::test]
async fn extra_body_adds_defaults_without_overriding_the_request()
-> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index d660f8c98..394046c15 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -99,7 +99,7 @@ pub async fn run(
.ok()
.and_then(Response::served_model)
.unwrap_or(&selected_model_id);
- let served_model = served_model.clone();
+ let served_model = clients.provider_model(served_model).clone();
metrics::observe_routed_request(&algorithm_name, &served_model, None, result)
} else {
let observe = |observation| {
@@ -397,12 +397,13 @@ impl CallPhase {
switchyard.candidate = index + 1,
switchyard.candidate_count = count,
switchyard.call_phase = phase.as_str(),
- selected_model = %model_id,
+ selected_model = %clients.provider_model(model_id),
+ selected_target = %model_id,
otel.kind = "client",
- otel.name = %format_args!("chat {model_id}"),
+ otel.name = %format_args!("chat {}", clients.provider_model(model_id)),
openinference.span.kind = "LLM",
gen_ai.operation.name = "chat",
- gen_ai.request.model = %model_id,
+ gen_ai.request.model = %clients.provider_model(model_id),
gen_ai.request.stream = tracing::field::Empty,
gen_ai.request.temperature = tracing::field::Empty,
gen_ai.request.top_p = tracing::field::Empty,
@@ -475,7 +476,12 @@ async fn call_one(
let result = if is_routing {
result
} else {
- metrics::observe_routed_request(algorithm, model_id, Some(duration), result)
+ metrics::observe_routed_request(
+ algorithm,
+ clients.provider_model(model_id),
+ Some(duration),
+ result,
+ )
};
observe(ModelCallObservation {
selected_model: model_id.clone(),
@@ -839,6 +845,12 @@ impl ClientRouter {
self
}
+ /// Resolves a target for model telemetry, including calls that fail before a response.
+ fn provider_model<'a>(&'a self, target: &'a ModelId) -> &'a ModelId {
+ self.route(target)
+ .map_or(target, |client| client.provider_model(target))
+ }
+
/// The client that serves `model`.
///
/// Errors with [`LlmClientError::Configuration`] when the router maps models and has no
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs
index 558b7cb5e..99fac9c3b 100644
--- a/crates/libsy-llm-client/tests/observability.rs
+++ b/crates/libsy-llm-client/tests/observability.rs
@@ -2295,3 +2295,142 @@ async fn in_flight_gauge_clears_when_a_run_is_abandoned() -> switchyard_libsy::R
);
Ok(())
}
+
+/// Routing keys remain distinct while both routing-time and answer metrics aggregate by provider.
+#[tokio::test]
+async fn target_aliases_preserve_provider_model_metric_labels() -> switchyard_libsy::Result<()> {
+ let _guard = serialize_test().lock().await;
+ let (store, exporter, provider, span_exporter, _) = telemetry();
+ struct MappedClient {
+ model: ModelId,
+ }
+ #[async_trait]
+ impl RoutedLlmClient for MappedClient {
+ fn provider_model<'a>(&'a self, _: &'a ModelId) -> &'a ModelId {
+ &self.model
+ }
+ async fn call(&self, request: Request) -> Result<Response, LlmClientError> {
+ UsageClient {
+ usage: Usage::default(),
+ }
+ .call(request)
+ .await
+ }
+ }
+ for routing_time in [false, true] {
+ let name = if routing_time {
+ "mapped-routing"
+ } else {
+ "mapped-answer"
+ };
+ let provider_model = ModelId::from(format!("provider/{name}"));
+ let client = Arc::new(MappedClient {
+ model: provider_model.clone(),
+ });
+ let models = Arc::new(
+ RuntimeModels::default().with_provider_models(
+ ["efficient", "capable"]
+ .into_iter()
+ .map(|target| (ModelId::from(target), provider_model.clone()))
+ .chain([("fallback".into(), "provider/fallback".into())])
+ .collect(),
+ ),
+ );
+ for target in ["efficient", "capable"] {
+ let algorithm = if routing_time {
+ Arc::new(RoutingCallAlgo {
+ name: name.into(),
+ target: target.into(),
+ }) as Arc<dyn Algorithm>
+ } else {
+ Arc::new(SingleCallAlgo {
+ name: name.into(),
+ target_set: vec![target.into(), "fallback".into(), "provider/direct".into()],
+ metadata: None,
+ })
+ };
+ let (selected, _) = switchyard_llm_client::run(
+ algorithm,
+ ClientRouter::single(client.clone()),
+ Request {
+ llm_request: text_request(None, "hello"),
+ ..Request::default()
+ },
+ models.clone(),
+ None,
+ )
+ .await?;
+ assert_eq!(selected, target);
+ let spans = store
+ .spans()
+ .into_iter()
+ .rev()
+ .filter(|span| span.fields.get("algorithm").map(String::as_str) == Some(name))
+ .collect::<Vec<_>>();
+ let span = find_span(&spans, "libsy.client_call", "selected_target", target);
+ assert_eq!(span.fields["selected_model"], provider_model.as_str());
+ assert_eq!(span.fields["gen_ai.request.model"], provider_model.as_str());
+ assert_eq!(span.fields["otel.name"], format!("chat {provider_model}"));
+ let run_span = find_span(&spans, "libsy.run", "algorithm", name);
+ let exported = span_exporter.get_finished_spans().expect("exported spans");
+ let exported_run = exported
+ .iter()
+ .find(|span| {
+ span.name == "libsy.run"
+ && otel_attribute(span, "outcome_id")
+ .is_some_and(|id| id.as_str() == run_span.fields["outcome_id"])
+ })
+ .expect("outcome span exported");
+ let mut expected_models = vec![provider_model.to_string().into()];
+ if !routing_time {
+ expected_models.extend(["provider/fallback".into(), "provider/direct".into()]);
+ }
+ assert_eq!(
+ otel_attribute(exported_run, "selected_model_ids"),
+ Some(&OtelValue::Array(OtelArray::String(expected_models)))
+ );
+ }
+ let exported = find_otel_span(
+ span_exporter,
+ &format!("chat {provider_model}"),
+ &provider_model,
+ );
+ assert_eq!(
+ otel_attribute(&exported, "selected_model"),
+ Some(&OtelValue::String(provider_model.to_string().into()))
+ );
+ let snapshots = flushed_metrics(exporter, provider);
+ for metric in ["switchyard.decisions", "switchyard.llm_calls"] {
+ assert_eq!(
+ u64_counter_value(
+ &snapshots,
+ metric,
+ &[
+ ("algorithm", name),
+ ("selected_model", provider_model.as_str())
+ ]
+ ),
+ Some(2)
+ );
+ for target in ["efficient", "capable"] {
+ assert_eq!(
+ u64_counter_value(
+ &snapshots,
+ metric,
+ &[("algorithm", name), ("selected_model", target)]
+ ),
+ None
+ );
+ }
+ }
+ assert_eq!(
+ u64_counter_value(
+ &snapshots,
+ "switchyard.requests",
+ &[("model", provider_model.as_str())]
+ ),
+ Some(2)
+ );
+ }
+ Ok(())
+}
diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs
index 06936a8ed..55cc17ffb 100644
--- a/crates/libsy/src/algorithms/stage.rs
+++ b/crates/libsy/src/algorithms/stage.rs
@@ -98,7 +98,7 @@ impl Classifier for SourceStamp {
let (classification, served) = self.inner.score(state, request, driver).await?;
// An abstaining classifier passes the turn on, so it is not its to claim.
if let Some(winner) = classification.argmax(false)? {
- record_routing_decision(self.source, &winner.target);
+ record_routing_decision(self.source, driver.provider_model(&winner.target));
driver.set_evidence_if_empty(serde_json::json!({
"source": self.source.as_str(),
}));
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
index 230db1856..67619622b 100644
--- a/crates/libsy/src/algorithms/util/llm_judge.rs
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -288,7 +288,7 @@ where
) -> &'static str {
let judge_target = driver
.first_model_for(&Category::Judge)
- .map(|c| c.as_str())
+ .map(|c| driver.provider_model(c).as_str())
.unwrap_or("missing");
report_fail_open(judge_target, error, reason);
if self.evidence.is_some() {
diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs
index d60760976..2d1fcbf7c 100644
--- a/crates/libsy/src/algorithms/util/stage.rs
+++ b/crates/libsy/src/algorithms/util/stage.rs
@@ -738,7 +738,7 @@ impl Classifier for StageClassifier {
] {
if Self::consume_hold(state, &Self::agent_key(request, key)) {
let target = driver.first_model_for(&category)?.clone();
- record_routing_decision(source, &target);
+ record_routing_decision(source, driver.provider_model(&target));
driver.set_evidence(serde_json::json!({"source": source.as_str()}));
return Ok((
Classification::Scores(vec![Score {
@@ -783,7 +783,7 @@ impl Classifier for StageClassifier {
{
Self::set_hold(state, &capable_hold_key, self.capable_hold_turns);
}
- record_routing_decision(source, target);
+ record_routing_decision(source, driver.provider_model(target));
if source == DecisionSource::Override
&& let Some(reason) = override_reason(&signal)
{
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index b4032c6a4..278c8b4e6 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -52,6 +52,7 @@ pub struct RuntimeModels {
by_category: HashMap<Category, Vec<ModelId>>,
subagent: Option<HashMap<Category, Vec<ModelId>>>,
has_target_restriction: bool,
+ provider_models: HashMap<ModelId, ModelId>,
}
impl RuntimeModels {
@@ -61,9 +62,21 @@ impl RuntimeModels {
by_category,
subagent: None,
has_target_restriction: false,
+ provider_models: HashMap::new(),
}
}
+ /// Supplies provider identities for telemetry while routing keeps using target keys.
+ pub fn with_provider_models(mut self, models: HashMap<ModelId, ModelId>) -> Self {
+ self.provider_models = models;
+ self
+ }
+
+ /// Returns the provider model for a routing key, or the key for direct library callers.
+ pub fn provider_model<'a>(&'a self, target: &'a ModelId) -> &'a ModelId {
+ self.provider_models.get(target).unwrap_or(target)
+ }
+
/// Adds the groups used for delegated sub-agent work.
pub fn with_subagent(mut self, models: HashMap<Category, Vec<ModelId>>) -> Self {
self.subagent = Some(models);
@@ -156,6 +169,7 @@ pub struct CallModel {
pub models: Vec<ModelId>,
/// Return client errors to the algorithm so it can apply its fallback policy.
pub recover_errors: bool,
+ telemetry_model: ModelId,
/// How to send the response back to the algorithm. `None` once the call is recorded.
reply: Option<oneshot::Sender<Result<Response>>>,
started: Instant,
@@ -207,10 +221,7 @@ impl CallModel {
fn record(&self, is_ok: bool) {
observability::record_llm_call(
&self.algorithm,
- self.models
- .first()
- .map(ModelId::as_str)
- .unwrap_or("NoTargets"),
+ self.telemetry_model.as_str(),
self.started.elapsed(),
is_ok,
);
@@ -497,7 +508,7 @@ impl Driver {
skip_all,
fields(
algorithm = self.algorithm,
- selected_model = %models.first().map(ModelId::as_str).unwrap_or("NoTargets"),
+ selected_model = %models.first().map(|model| self.provider_model(model).as_str()).unwrap_or("NoTargets"),
openinference.span.kind = "CHAIN",
outcome = tracing::field::Empty,
input_tokens = tracing::field::Empty,
@@ -522,6 +533,7 @@ impl Driver {
let call = CallModel {
algorithm: self.algorithm.clone(),
request,
+ telemetry_model: self.provider_model(selected_model_id).clone(),
models,
recover_errors,
reply: Some(reply),
@@ -581,6 +593,11 @@ impl Driver {
.map_err(|_| LibsyError::from(DriverError::ResponseDropped))?
}
+ /// Provider identity for telemetry; algorithms still route on the original target key.
+ pub fn provider_model<'a>(&'a self, target: &'a ModelId) -> &'a ModelId {
+ self.models.provider_model(target)
+ }
+
/// The available models for this category, typically ordered best-first.
pub fn models_for(&self, category: &Category) -> &[ModelId] {
if *category == Category::Judge
@@ -635,7 +652,12 @@ impl Driver {
crate::OutcomeMetadata::new(self.algorithm.clone(), evidence.outcome)
});
metadata.judgments.extend(evidence.judgments);
- observability::record_outcome(metadata, &outcome.selected_model_ids);
+ let provider_models = outcome
+ .selected_model_ids
+ .iter()
+ .map(|target| self.provider_model(target).clone())
+ .collect::<Vec<_>>();
+ observability::record_outcome(metadata, &provider_models);
outcome
});
let selected_model = result
@@ -648,7 +670,7 @@ impl Driver {
.await
.map_err(|_| DriverError::StreamClosed)?;
if let Some(selected_model) = selected_model {
- observability::record_decision(&self.algorithm, &selected_model);
+ observability::record_decision(&self.algorithm, self.provider_model(&selected_model));
}
Ok(())
}
diff --git a/crates/protocol/src/client.rs b/crates/protocol/src/client.rs
index f791bd8f9..dd5f3a977 100644
--- a/crates/protocol/src/client.rs
+++ b/crates/protocol/src/client.rs
@@ -159,6 +159,11 @@ impl RoutingFallbackReason {
/// not serialize requests unless their transport requires it.
#[async_trait]
pub trait RoutedLlmClient: Send + Sync {
+ /// Resolves a routing key for telemetry. Direct model clients keep the key unchanged.
+ fn provider_model<'a>(&'a self, target: &'a ModelId) -> &'a ModelId {
+ target
+ }
+
/// Make a request
async fn call(&self, request: Request) -> Result<Response, LlmClientError>;
}
diff --git a/crates/protocol/src/model_id.rs b/crates/protocol/src/model_id.rs
index 854e9d77c..78de651fa 100644
--- a/crates/protocol/src/model_id.rs
+++ b/crates/protocol/src/model_id.rs
@@ -14,8 +14,8 @@ use serde::{Deserialize, Serialize};
/// It can name a provider model, such as `"openai/gpt-oss-20b"`. It can also name a
/// Switchyard route, such as `"switchyard/random"`.
///
-/// A target name from server config, such as `"capable"`, is not a model ID. The server
-/// resolves target names to model IDs before routing.
+/// The runner routes on target names from server config, such as `"capable"`, and sends
+/// each target's provider model upstream.
///
/// This type acts like a string. It can be printed, compared, and saved as a JSON string.
#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs
index fb0f4048d..39d0b2ef5 100644
--- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs
+++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs
@@ -254,20 +254,26 @@ impl SwitchyardRuntime {
.push(observation);
});
match route.execute(request, Some(observer)).await {
- Ok(output) => {
+ Ok(mut output) => {
let outcome_fields = self.emit_observations(
&mut events,
take_observations(&observations),
&metadata,
+ route,
);
- let served_model = output.response.served_model().map(|model| model.as_str());
+ let served_target = output.response.served_model().cloned();
+ let served_model = served_target
+ .as_ref()
+ .map(|target| route.provider_model(target));
let mut decision_outcome_fields = outcome_fields.clone();
let mut data = json!({
"algorithm": route.algorithm_name(),
- "selected_model": output.selected_model.as_str(),
+ "selected_model": route.provider_model(&output.selected_model).as_str(),
"served_model": served_model,
- "fallback_used": served_model
- .map(|model| model != output.selected_model.as_str()),
+ "selected_target": output.selected_model.as_str(),
+ "served_target": served_target,
+ "reasoning_effort": served_target.as_ref().and_then(|target| route.target(target)).and_then(|target| target.reasoning_effort.as_deref()),
+ "fallback_used": served_target.as_ref().map(|target| target != &output.selected_model),
});
if let Json::Object(data) = &mut data {
data.append(&mut decision_outcome_fields);
@@ -278,6 +284,9 @@ impl SwitchyardRuntime {
metadata,
severity: Some(LogSeverity::Info),
}));
+ if let Some(model) = served_model {
+ output.response.set_served_model(model);
+ }
Execution {
result: Ok(RoutedResponse {
response: output.response,
@@ -291,6 +300,7 @@ impl SwitchyardRuntime {
&mut events,
take_observations(&observations),
&metadata,
+ route,
);
self.route_execution_error_mark(
&mut events,
@@ -319,10 +329,20 @@ impl SwitchyardRuntime {
events: &mut Vec<RoutingEvent>,
observations: Vec<RunObservation>,
metadata: &Json,
+ route: &Route,
) -> Map<String, Json> {
let mut call_index = 0;
let mut outcome_fields = Map::new();
- for observation in observations {
+ for mut observation in observations {
+ // Relay's model fields and token metrics retain provider identities.
+ match &mut observation {
+ RunObservation::LlmCall(call)
+ | RunObservation::DecisionCall(call)
+ | RunObservation::AnswerCall(call) => {
+ call.selected_model = route.provider_model(&call.selected_model).clone();
+ }
+ _ => {}
+ }
match observation {
RunObservation::Outcome(outcome) => {
outcome_fields.insert(
@@ -1062,7 +1082,9 @@ mod tests {
})
.expect("decision mark should be emitted");
assert_eq!(decision["selected_model"], "target/model");
+ assert_eq!(decision["selected_target"], "default");
assert_eq!(decision["served_model"], "target/model");
+ assert_eq!(decision["served_target"], "default");
assert_eq!(decision["fallback_used"], false);
assert!(decision["outcome_id"].is_string());
let response = execution.result.expect("target call should succeed");
@@ -1071,6 +1093,79 @@ mod tests {
}
}
+ // Falling back between efforts must remain visible even when provider IDs match.
+ #[tokio::test]
+ async fn same_model_fallback_keeps_provider_fields_and_target_details() {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(body_partial_json(json!({"reasoning_effort": "medium"})))
+ .respond_with(ResponseTemplate::new(503))
+ .expect(1)
+ .mount(&server)
+ .await;
+ Mock::given(method("POST"))
+ .and(body_partial_json(json!({"model": "shared/model", "reasoning_effort": "max"})))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "id": "reply", "model": "shared/model",
+ "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
+ "usage": {"prompt_tokens": 2, "completion_tokens": 1}
+ }))).expect(1).mount(&server).await;
+ let runner = Runner::from_toml(&format!(
+ r#"
+schema_version = 1
+[llm_clients.upstream]
+format = "openai_chat"
+base_url = "{}/v1"
+max_retries = 0
+[targets.efficient]
+id = "shared/model"
+llm_client = "upstream"
+reasoning_effort = "medium"
+[targets.capable]
+id = "shared/model"
+llm_client = "upstream"
+reasoning_effort = "max"
+[routes.test]
+id = "route"
+type = "random"
+targets = ["efficient", "capable"]
+weights = [1000000, 1]
+seed = 17
+"#,
+ server.uri()
+ ))
+ .unwrap();
+ let runtime = SwitchyardRuntime {
+ runner,
+ translation: TranslationEngine::default(),
+ redactor: Arc::default(),
+ };
+ let request = runtime.decode_request(WireFormat::OpenAiChat, RelayRequest {
+ headers: Map::new(), content: json!({"model": "route", "messages": [{"role": "user", "content": "hello"}]}),
+ }, false).unwrap();
+ let execution = runtime
+ .execute_buffered(WireFormat::OpenAiChat, request)
+ .await;
+ assert_eq!(execution.result.unwrap()["model"], "shared/model");
+ let mut saw_decision = false;
+ for event in execution.events {
+ if let RoutingEvent::Mark(mark) = event {
+ if mark.name == "switchyard.routing.decision" {
+ saw_decision = true;
+ assert_eq!(mark.data["selected_model"], "shared/model");
+ assert_eq!(mark.data["served_model"], "shared/model");
+ assert_eq!(mark.data["selected_target"], "efficient");
+ assert_eq!(mark.data["served_target"], "capable");
+ assert_eq!(mark.data["reasoning_effort"], "max");
+ assert_eq!(mark.data["fallback_used"], true);
+ } else if mark.name == "switchyard.routing.llm_call" {
+ assert_eq!(mark.data["selected_model"], "shared/model");
+ }
+ }
+ }
+ assert!(saw_decision);
+ }
+
// Routing succeeds before answer candidates exhaust, so the error keeps its metadata.
#[tokio::test]
async fn failed_answer_keeps_routing_outcome_in_error_mark() {
@@ -1331,6 +1426,7 @@ mod tests {
RunObservation::RoutingOverhead(std::time::Duration::from_millis(3)),
],
&json!({"session_id": "session"}),
+ runtime.runner.route("switchyard").unwrap(),
);
assert_eq!(events.len(), 6);
@@ -1490,6 +1586,7 @@ mod tests {
}),
],
&json!({}),
+ runtime.runner.route("switchyard").unwrap(),
);
assert_eq!(events.len(), 3);
diff --git a/crates/switchyard-runner/src/algorithm_factory.rs b/crates/switchyard-runner/src/algorithm_factory.rs
index eb92af4f8..647985530 100644
--- a/crates/switchyard-runner/src/algorithm_factory.rs
+++ b/crates/switchyard-runner/src/algorithm_factory.rs
@@ -30,7 +30,8 @@ impl AlgorithmFactory {
/// Builds algorithms and their model groups, keyed by configured route `id`.
///
- /// The host serves model calls and owns target prompts, provider settings and transport.
+ /// LLM calls and outcomes use target names; the host resolves their provider IDs and
+ /// owns target prompts, provider settings and transport. Decision calls use provider IDs.
/// Reuse the returned algorithms across requests to retain algorithm-owned state.
pub fn build(&self) -> Result<BTreeMap<String, AlgorithmAndModels>, RunnerError> {
self.config.validate()?;
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs
index 5515bc177..68234315d 100644
--- a/crates/switchyard-runner/src/config.rs
+++ b/crates/switchyard-runner/src/config.rs
@@ -275,6 +275,7 @@ impl DeploymentConfig {
format: client.format.wire_format(),
base_url: client.base_url.as_str().to_string(),
extra_body: target.extra_body.clone(),
+ reasoning_effort: target.reasoning_effort.clone(),
})
}
@@ -314,6 +315,13 @@ impl DeploymentConfig {
"target {name} is defined as both an LLM and decision target"
)));
}
+ // Algorithms route LLM targets by name and decision targets by id.
+ if self.targets.contains_key(target.id.as_str()) {
+ return Err(RunnerError::configuration(format!(
+ "decision target {name} id {} is also an LLM target name; rename one of them",
+ target.id
+ )));
+ }
if !self.decision_clients.contains_key(&target.decision_client) {
return Err(RunnerError::configuration(format!(
"decision target {name} references unknown decision client {}",
@@ -365,6 +373,17 @@ impl DeploymentConfig {
.algorithm
.build_with_runtime_models(route_name, &targets)
.map_err(|error| RunnerError::configuration_source(error.to_string(), error))?;
+ let models = models.with_provider_models(
+ config
+ .callable_target_names()
+ .into_iter()
+ .filter_map(|name| {
+ self.targets
+ .get(name)
+ .map(|target| (ModelId::from(name), target.id.clone()))
+ })
+ .collect(),
+ );
if let Some(infra) = &config.infra {
if !matches!(&config.algorithm, AlgorithmSpec::LlmClassifier { config }
if matches!(config.mode, None | Some(crate::ClassifierMode::Capability))
@@ -385,37 +404,6 @@ impl DeploymentConfig {
fn build(self) -> RunnerResult<Runner> {
self.validate()?;
- // The LLM client keeps one backend per model id, so two targets naming the same model on
- // the same client share it. That is harmless when their request settings agree (an alias
- // for a different system prompt, say) and silently wrong when they do not: the second
- // target's reasoning_effort or extra_body would never reach the wire.
- let mut seen_client_model_ids: HashMap<(&str, &str), (&String, &TargetConfig)> =
- HashMap::new();
- for (target_name, target) in &self.targets {
- match seen_client_model_ids.entry((target.llm_client.as_str(), target.id.as_str())) {
- std::collections::hash_map::Entry::Vacant(slot) => {
- slot.insert((target_name, target));
- }
- std::collections::hash_map::Entry::Occupied(slot) => {
- let (first_name, first) = slot.get();
- if first.reasoning_effort != target.reasoning_effort
- || first.extra_body != target.extra_body
- || first.omit_body_fields != target.omit_body_fields
- {
- return Err(RunnerError::configuration(format!(
- "targets {first_name} and {target_name} both name model {} on llm client {} but with different reasoning_effort, extra_body, or omit_body_fields; one target per model id is kept, so give each its own model id or llm client",
- target.id, target.llm_client
- )));
- }
- tracing::warn!(
- "target {target_name} reuses model id {} on llm client {}; only one target per id is kept and the other is dropped. Give each target a unique model id, or point both routes at one target.",
- target.id,
- target.llm_client
- );
- }
- }
- }
-
let mut provider_api_keys = Vec::new();
let clients = self.build_clients(&mut provider_api_keys)?;
let decision_clients = self.build_decision_clients(&mut provider_api_keys)?;
@@ -437,8 +425,9 @@ impl DeploymentConfig {
self.build_anthropic_auxiliary_target(config, &clients);
let responses_auxiliary_target =
self.build_responses_auxiliary_target(config, &clients);
- let decision_targets = config
- .routing_target_names()
+ // Judges are included so a judge answer maps to its provider model too.
+ let route_targets = config
+ .callable_target_names()
.into_iter()
.filter_map(|name| self.decision_target(name))
.collect();
@@ -449,7 +438,7 @@ impl DeploymentConfig {
capabilities,
anthropic_auxiliary_target,
responses_auxiliary_target,
- decision_targets,
+ route_targets,
models,
)
.with_infra(config.infra.clone());
@@ -513,15 +502,19 @@ impl DeploymentConfig {
"target {target_name} reasoning_effort must not be empty"
)));
}
- if matches!(client_config.format, ClientFormat::AnthropicMessages) {
+ // Anthropic rejects any other value with a 400 on every request.
+ if matches!(client_config.format, ClientFormat::AnthropicMessages)
+ && !ANTHROPIC_EFFORTS.contains(&effort.as_str())
+ {
return Err(RunnerError::configuration(format!(
- "target {target_name} reasoning_effort is only supported on openai_chat and openai_responses clients"
+ "target {target_name} reasoning_effort {effort:?} is not an anthropic_messages effort; use one of {}",
+ ANTHROPIC_EFFORTS.join(", ")
)));
}
}
model_configs.push(
ModelConfig::new(
- target.id.clone(),
+ target_name.as_str(),
build_backend(
&target.llm_client,
client_config,
@@ -531,7 +524,8 @@ impl DeploymentConfig {
)?,
None,
)
- .with_responses_reasoning(client_config.responses_reasoning.unwrap_or_default()),
+ .with_responses_reasoning(client_config.responses_reasoning.unwrap_or_default())
+ .with_provider_model(target.id.clone()),
);
}
@@ -585,10 +579,13 @@ impl DeploymentConfig {
.collect()
}
+ /// Algorithms route on LLM target names, so targets that share a provider model keep
+ /// their own settings. The LLM client sends each target's `id` upstream. Decision targets
+ /// route on their `id`, which the decision client sends as is.
fn build_targets(&self) -> BTreeMap<String, ModelId> {
self.targets
- .iter()
- .map(|(name, config)| (name.clone(), config.id.clone()))
+ .keys()
+ .map(|name| (name.clone(), ModelId::from(name.as_str())))
.chain(
self.decision_targets
.iter()
@@ -612,7 +609,6 @@ impl DeploymentConfig {
routing_answer_target,
} = self.build_route_target_prompts(route_name, route)?;
let mut by_model = HashMap::new();
- let mut targets_by_model: HashMap<&str, (&str, &TargetConfig)> = HashMap::new();
let mut caller_auth = None;
let mut has_mixed_families = false;
let mut forwarding_origins = BTreeSet::new();
@@ -630,14 +626,6 @@ impl DeploymentConfig {
let target = self.targets.get(name).ok_or_else(|| {
RunnerError::configuration(format!("route references unknown target {name}"))
})?;
- if let Some((first_name, first)) = targets_by_model.insert(&target.id, (name, target))
- && first.llm_client != target.llm_client
- {
- return Err(RunnerError::configuration(format!(
- "route {route_name} targets {first_name} and {name} use model {} on different llm clients; execution is keyed by model id, so use distinct model ids within this route or put these targets in separate routes",
- target.id
- )));
- }
let client = clients.get(&target.llm_client).ok_or_else(|| {
RunnerError::configuration(format!("target {name} has no constructed llm client"))
})?;
@@ -654,7 +642,7 @@ impl DeploymentConfig {
forwarding_origins.insert(client_config.base_url.0.origin().ascii_serialization());
}
let client: Arc<dyn RoutedLlmClient> = client.clone();
- by_model.insert(target.id.clone(), client);
+ by_model.insert(ModelId::from(name), client);
}
// Only forwarding clients count: an api_key_env client sends the server's own key.
// Forwarding clients must share one credential family unless they all use the same
@@ -673,7 +661,7 @@ impl DeploymentConfig {
let completion_targets = route
.routing_target_names()
.into_iter()
- .map(|name| self.targets[name].id.clone())
+ .map(ModelId::from)
.collect::<Vec<_>>();
let router = ClientRouter::new_with_decision_clients(
by_model,
@@ -689,7 +677,7 @@ impl DeploymentConfig {
.filter_map(|name| {
let target = &self.targets[name];
(!self.llm_clients[&target.llm_client].fallback_on_rate_limit)
- .then(|| target.id.clone())
+ .then(|| ModelId::from(name))
})
.collect(),
);
@@ -703,23 +691,12 @@ impl DeploymentConfig {
route: &RouteConfig,
) -> RunnerResult<TargetPromptPolicy> {
let mut prompts = HashMap::new();
- let mut aliases = HashMap::<&ModelId, Option<&str>>::new();
for name in route.algorithm.routing_target_names() {
let target = self.targets.get(name).ok_or_else(|| {
RunnerError::configuration(format!("route references unknown target {name}"))
})?;
- let prompt = target.system_prompt.as_deref();
- if aliases
- .insert(&target.id, prompt)
- .is_some_and(|configured| configured != prompt)
- {
- return Err(RunnerError::configuration(format!(
- "route {route_name} maps completion target aliases to model {} with different system_prompt values",
- target.id
- )));
- }
- if let Some(prompt) = prompt {
- prompts.insert(target.id.clone(), prompt.to_string());
+ if let Some(prompt) = &target.system_prompt {
+ prompts.insert(ModelId::from(name), prompt.clone());
}
}
let mut policy = TargetPromptPolicy {
@@ -731,22 +708,15 @@ impl DeploymentConfig {
else {
return Ok(policy);
};
- let response = self.targets.get(response_name).ok_or_else(|| {
- RunnerError::configuration(format!("route references unknown target {response_name}"))
- })?;
- if !policy.prompts.contains_key(&response.id) {
+ if !policy.prompts.contains_key(response_name) {
return Ok(policy);
}
- let dependency = self.targets.get(dependency_name).ok_or_else(|| {
- RunnerError::configuration(format!("route references unknown target {dependency_name}"))
- })?;
- if response.id == dependency.id {
+ if response_name == dependency_name {
return Err(RunnerError::configuration(format!(
- "route {route_name} cannot apply system_prompt to target {response_name}: model {} is also used by routing-only target {dependency_name}",
- response.id,
+ "route {route_name} cannot apply system_prompt to target {response_name}: it is also the routing-only target"
)));
}
- policy.routing_answer_target = Some(response.id.clone());
+ policy.routing_answer_target = Some(ModelId::from(response_name));
Ok(policy)
}
@@ -777,7 +747,7 @@ impl DeploymentConfig {
}
let client = clients.get(&efficient_target.llm_client)?;
Some(ModelPropertiesProbe {
- model: efficient_target.id.clone(),
+ model: ModelId::from(efficient_target_name),
format: client_config.format.wire_format(),
client: Arc::clone(client),
})
@@ -798,7 +768,11 @@ impl DeploymentConfig {
clients,
AuxiliaryOperation::AnthropicCountTokens,
)?;
- Some((count_tokens_priority(name, &target.model), index, target))
+ Some((
+ count_tokens_priority(name, &self.targets[name].id),
+ index,
+ target,
+ ))
})
.min_by_key(|(priority, index, _)| (*priority, *index))
.map(|(_, _, target)| target)
@@ -822,15 +796,19 @@ impl DeploymentConfig {
) -> Option<AuxiliaryTarget> {
let target = self.targets.get(name)?;
let client = clients.get(&target.llm_client)?;
+ let model = ModelId::from(name);
client
- .supports_auxiliary(&target.id, operation)
+ .supports_auxiliary(&model, operation)
.then(|| AuxiliaryTarget {
- model: target.id.clone(),
+ model,
client: client.clone(),
})
}
}
+/// Values Anthropic accepts for `output_config.effort`.
+const ANTHROPIC_EFFORTS: [&str; 5] = ["low", "medium", "high", "xhigh", "max"];
+
fn count_tokens_priority(target_name: &str, model_id: &ModelId) -> usize {
let target_name = target_name.to_ascii_lowercase();
let model_id = model_id.to_ascii_lowercase();
@@ -937,7 +915,6 @@ struct TargetConfig {
omit_body_fields: BTreeSet<String>,
system_prompt: Option<String>,
/// Reasoning effort forced on every request to this target, replacing the caller's value.
- /// Only meaningful on `openai_chat` and `openai_responses` clients.
reasoning_effort: Option<String>,
}
@@ -1204,30 +1181,30 @@ efficient_target = "weak"
let models = classifier.models();
assert_eq!(
models.models_for(&Category::Judge),
- [ModelId::from("classifier/model")]
+ [ModelId::from("classifier")]
);
assert_eq!(
models.models_for(&Category::Efficient),
- [ModelId::from("weak/model")]
+ [ModelId::from("weak")]
);
assert_eq!(
models.models_for(&Category::Capable),
- [ModelId::from("strong/model")]
+ [ModelId::from("strong")]
);
assert_eq!(
models.models_for(&Category::Any),
- [ModelId::from("weak/model"), ModelId::from("strong/model")]
+ [ModelId::from("weak"), ModelId::from("strong")]
);
let plan_execute = runner
.route("switchyard/plan-execute")
.expect("plan-execute route should exist");
assert_eq!(
plan_execute.models().models_for(&Category::Capable),
- [ModelId::from("strong/model")]
+ [ModelId::from("strong")]
);
assert_eq!(
plan_execute.models().models_for(&Category::Efficient),
- [ModelId::from("weak/model")]
+ [ModelId::from("weak")]
);
assert!(runner.route("switchyard/passthrough").is_some());
Ok(())
@@ -1272,10 +1249,7 @@ target = "strong"
})],
});
- for (config, expected) in [
- (VALID_CONFIG, "strong/model"),
- (configured.as_str(), "weak/model"),
- ] {
+ for (config, expected) in [(VALID_CONFIG, "strong"), (configured.as_str(), "weak")] {
let runner = Runner::from_toml(config)?;
let route = runner.route("switchyard/plan-execute").unwrap();
let outcome = route.decide(request.clone()).await?;
@@ -1408,11 +1382,11 @@ classify_trigger = "new_session""#,
assert_eq!(
models.subagent_models_for(&Category::Any),
- [ModelId::from("strong/model")]
+ [ModelId::from("strong")]
);
assert_eq!(
models.models_for(&Category::Any),
- [ModelId::from("strong/model"), ModelId::from("weak/model")]
+ [ModelId::from("strong"), ModelId::from("weak")]
);
Ok(())
}
@@ -1661,31 +1635,13 @@ classify_trigger"#,
Ok(())
}
+ // A routing-only judge cannot inherit the answer target's system prompt.
#[test]
- fn aliased_completion_targets_reject_prompt_conflicts() {
- let configured = stage_config()
- .replace(
- "id = \"strong/model\"\nllm_client = \"responses\"",
- "id = \"strong/model\"\nllm_client = \"responses\"\nsystem_prompt = \"capable\"",
- )
- .replace(
- "[routes.stage]",
- "[targets.strong_alias]\nid = \"strong/model\"\nllm_client = \"responses\"\n\n[routes.stage]",
- )
- .replace("efficient_target = \"weak\"", "efficient_target = \"strong_alias\"");
- let message = error_message(&configured);
- assert!(
- message.contains("completion target aliases to model strong/model with different system_prompt values"),
- "unexpected error: {message}"
- );
- }
-
- #[test]
- fn prompted_routing_response_cannot_share_a_model_with_a_dependency() {
+ fn rejects_prompted_answer_target_used_as_routing_only_target() {
let configured = VALID_CONFIG
.replace(
- "id = \"classifier/model\"\nllm_client = \"primary\"",
- "id = \"weak/model\"\nllm_client = \"primary\"",
+ "classifier_target = \"classifier\"",
+ "classifier_target = \"weak\"",
)
.replace(
"id = \"weak/model\"\nllm_client = \"anthropic\"",
@@ -1696,11 +1652,9 @@ classify_trigger"#,
"base_threshold = 0.5\nescalation = { confirmations = 1 }",
);
- let message = error_message(&configured);
-
- assert!(
- message.contains("cannot apply system_prompt to target weak: model weak/model is also used by routing-only target classifier"),
- "unexpected error: {message}"
+ assert_eq!(
+ error_message(&configured),
+ "route classifier cannot apply system_prompt to target weak: it is also the routing-only target"
);
}
@@ -1781,7 +1735,7 @@ classify_trigger"#,
}
#[test]
- fn a_target_reasoning_effort_parses_and_is_rejected_where_unsupported() -> RunnerResult<()> {
+ fn a_target_reasoning_effort_parses_on_every_client_format() -> RunnerResult<()> {
let strong = "[targets.strong]\nid = \"strong/model\"\nllm_client = \"responses\"";
let weak = "[targets.weak]\nid = \"weak/model\"\nllm_client = \"anthropic\"";
assert!(VALID_CONFIG.contains(strong) && VALID_CONFIG.contains(weak));
@@ -1793,37 +1747,28 @@ classify_trigger"#,
assert!(error_message(&blank).contains("reasoning_effort must not be empty"));
let anthropic = VALID_CONFIG.replace(weak, &format!("{weak}\nreasoning_effort = \"high\""));
+ runner_from_toml(&anthropic)?;
+
+ let minimal =
+ VALID_CONFIG.replace(weak, &format!("{weak}\nreasoning_effort = \"minimal\""));
assert!(
- error_message(&anthropic)
- .contains("only supported on openai_chat and openai_responses")
+ error_message(&minimal).contains("is not an anthropic_messages effort"),
+ "{}",
+ error_message(&minimal)
);
Ok(())
}
#[test]
- fn duplicate_targets_with_conflicting_settings_are_rejected() -> RunnerResult<()> {
- let strong = "[targets.strong]\nid = \"strong/model\"\nllm_client = \"responses\"";
- assert!(VALID_CONFIG.contains(strong));
- // Same model, same client, different effort: the second target could never take effect.
- let conflicting = VALID_CONFIG.replace(
- strong,
- &format!(
- "{strong}\n\n[targets.strong_max]\nid = \"strong/model\"\nllm_client = \"responses\"\nreasoning_effort = \"max\""
- ),
+ fn decision_target_ids_must_not_match_llm_target_names() {
+ let configured = format!(
+ "{VALID_CONFIG}\n[decision_clients.decisions]\nformat = \"system_one\"\nendpoint = \"https://example.test/decisions\"\napi_key_env = \"PATH\"\ntimeout_ms = 5000\n\n[decision_targets.judge]\nid = \"strong\"\ndecision_client = \"decisions\"\n"
);
+ let message = error_message(&configured);
assert!(
- error_message(&conflicting)
- .contains("different reasoning_effort, extra_body, or omit_body_fields"),
- "{}",
- error_message(&conflicting)
- );
- // An alias with identical settings is still allowed (it only warns).
- let alias = VALID_CONFIG.replace(
- strong,
- &format!("{strong}\n\n[targets.strong_alias]\nid = \"strong/model\"\nllm_client = \"responses\""),
+ message.contains("decision target judge id strong is also an LLM target name"),
+ "{message}"
);
- runner_from_toml(&alias)?;
- Ok(())
}
#[test]
@@ -2155,7 +2100,7 @@ target = "smart"
#[test]
fn accepts_same_model_id_on_different_llm_clients() -> RunnerResult<()> {
- // Separate routes may serve the same model through different clients.
+ // One model may be served through different clients in separate routes.
const CROSS_PROVIDER: &str = r#"
schema_version = 1
@@ -2186,27 +2131,6 @@ type = "passthrough"
target = "azure"
"#;
runner_from_toml(CROSS_PROVIDER)?;
- let shared_route = format!(
- r#"{CROSS_PROVIDER}
-
-[routes.shared]
-id = "switchyard/shared"
-type = "stage_router"
-capable_target = "openai"
-efficient_target = "azure"
-picker = "efficient_first"
-confidence_threshold = 0.5
-"#
- );
- let message = error_message(&shared_route);
- assert!(
- message.contains("route shared")
- && message.contains("openai")
- && message.contains("azure")
- && message.contains("gpt-4o")
- && message.contains("different llm clients"),
- "{message}"
- );
Ok(())
}
@@ -2695,15 +2619,15 @@ advisor_target = "advisor"
// route's last-resort pool.
assert_eq!(
models.models_for(&Category::Efficient),
- [ModelId::from("executor/model")]
+ [ModelId::from("executor")]
);
assert_eq!(
models.models_for(&Category::Any),
- [ModelId::from("executor/model")]
+ [ModelId::from("executor")]
);
assert_eq!(
models.models_for(&Category::Judge),
- [ModelId::from("advisor/model")]
+ [ModelId::from("advisor")]
);
assert_eq!(
state
diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rs
index 0d263a3b7..2bf16de5e 100644
--- a/crates/switchyard-runner/src/route.rs
+++ b/crates/switchyard-runner/src/route.rs
@@ -3,6 +3,7 @@
//! One configured algorithm and the clients that serve its targets.
+use std::collections::HashMap;
use std::error::Error;
use std::sync::Arc;
@@ -108,6 +109,7 @@ impl RunnerError {
}
/// A configured algorithm and the per-target clients its calls resolve through.
+#[derive(Clone)]
pub struct Route {
algorithm: Arc<dyn Algorithm>,
// Resolves each offloaded call to the client configured for the target the algorithm
@@ -118,7 +120,7 @@ pub struct Route {
capabilities: ModelCapabilities,
anthropic_auxiliary_target: Option<AuxiliaryTarget>,
responses_auxiliary_target: Option<AuxiliaryTarget>,
- decision_targets: Vec<DecisionTarget>,
+ targets: Arc<HashMap<ModelId, DecisionTarget>>,
models: Arc<RuntimeModels>,
infra: Option<libsy::InfraPolicy>,
}
@@ -139,9 +141,19 @@ impl Route {
capabilities: ModelCapabilities,
anthropic_auxiliary_target: Option<AuxiliaryTarget>,
responses_auxiliary_target: Option<AuxiliaryTarget>,
- decision_targets: Vec<DecisionTarget>,
+ targets: Vec<DecisionTarget>,
models: RuntimeModels,
) -> Self {
+ let models = models.with_provider_models(
+ targets
+ .iter()
+ .map(|target| (ModelId::from(target.target.as_str()), target.model.clone()))
+ .collect(),
+ );
+ let targets = targets
+ .into_iter()
+ .map(|target| (ModelId::from(target.target.as_str()), target))
+ .collect();
Self {
algorithm,
clients,
@@ -149,7 +161,7 @@ impl Route {
capabilities,
anthropic_auxiliary_target,
responses_auxiliary_target,
- decision_targets,
+ targets: Arc::new(targets),
models: Arc::new(models),
infra: None,
}
@@ -170,12 +182,23 @@ impl Route {
self.caller_auth
}
+ /// Returns non-secret metadata for a configured LLM target, including judges.
+ pub fn target(&self, selected: &ModelId) -> Option<&DecisionTarget> {
+ self.targets.get(selected)
+ }
+
/// Resolves a selected model to this route's non-secret target metadata.
pub(crate) fn decision_target(&self, model: &ModelId) -> Option<DecisionTarget> {
- self.decision_targets
- .iter()
- .find(|target| target.model == *model)
- .cloned()
+ self.target(model).cloned()
+ }
+
+ /// The provider model that serves `selected`, or `selected` itself when it names no
+ /// target of this route.
+ pub fn provider_model<'a>(&'a self, selected: &'a ModelId) -> &'a ModelId {
+ self.targets
+ .get(selected)
+ .map(|target| &target.model)
+ .unwrap_or(selected)
}
/// Returns the models grouped for one algorithm execution.
diff --git a/crates/switchyard-runner/src/runner.rs b/crates/switchyard-runner/src/runner.rs
index 0ddf249b8..ad6b1d60d 100644
--- a/crates/switchyard-runner/src/runner.rs
+++ b/crates/switchyard-runner/src/runner.rs
@@ -47,6 +47,8 @@ pub struct DecisionTarget {
pub format: WireFormat,
pub base_url: String,
pub extra_body: BTreeMap<String, Value>,
+ /// Effort the target forces, so a caller that dispatches the call itself can apply it.
+ pub reasoning_effort: Option<String>,
}
pub(crate) struct ModelPropertiesProbe {
diff --git a/crates/switchyard-runner/tests/algorithm_factory.rs b/crates/switchyard-runner/tests/algorithm_factory.rs
index 0ee584d3c..a6512b2dd 100644
--- a/crates/switchyard-runner/tests/algorithm_factory.rs
+++ b/crates/switchyard-runner/tests/algorithm_factory.rs
@@ -62,20 +62,28 @@ fn full_deployment_builds_without_provider_credentials() {
);
assert_eq!(
models.models_for(&Category::Capable),
- [ModelId::from("upstream/capable")]
+ [ModelId::from("capable")]
);
assert_eq!(
models.models_for(&Category::Efficient),
- [ModelId::from("upstream/fast")]
+ [ModelId::from("fast")]
);
assert_eq!(
models.models_for(&Category::Any),
- [
- ModelId::from("upstream/fast"),
- ModelId::from("upstream/capable")
- ]
+ [ModelId::from("fast"), ModelId::from("capable")]
);
+ for (target, provider) in [
+ ("fast", "upstream/fast"),
+ ("capable", "upstream/capable"),
+ ("upstream/judge", "upstream/judge"),
+ ] {
+ assert_eq!(
+ models.provider_model(&ModelId::from(target)).as_str(),
+ provider
+ );
+ }
+
let error = Runner::from_toml(DEPLOYMENT)
.err()
.expect("the native runner still requires configured credentials");
@@ -89,16 +97,13 @@ async fn configured_parent_and_subagent_targets_stay_separate() {
.build()
.expect("algorithms should build");
let (algorithm, models) = algorithms.remove("public/main").expect("public route id");
- assert_eq!(
- models.models_for(&Category::Any),
- [ModelId::from("upstream/fast")]
- );
+ assert_eq!(models.models_for(&Category::Any), [ModelId::from("fast")]);
assert_eq!(
models.subagent_models_for(&Category::Any),
- [ModelId::from("upstream/capable")]
+ [ModelId::from("capable")]
);
let models = Arc::new(models);
- for (delegated, selected) in [(false, "upstream/fast"), (true, "upstream/capable")] {
+ for (delegated, selected) in [(false, "fast"), (true, "capable")] {
let request = Request {
llm_request: text_request(Some("public/main".to_string()), "hello"),
metadata: delegated.then(|| Metadata {
diff --git a/crates/switchyard-runner/tests/openai_decisions.rs b/crates/switchyard-runner/tests/openai_decisions.rs
index 8a40b53c9..d9544eff5 100644
--- a/crates/switchyard-runner/tests/openai_decisions.rs
+++ b/crates/switchyard-runner/tests/openai_decisions.rs
@@ -97,7 +97,14 @@ evidence = {{ description = "Compare the two profiles" }}
})),
)
.await?;
- assert_eq!(output.selected_model, "weak/model");
+ assert_eq!(output.selected_model, "weak");
+ assert_eq!(
+ runner
+ .route("switchyard/classifier")
+ .unwrap()
+ .provider_model(&output.selected_model),
+ "weak/model"
+ );
let usage = observations.try_iter().find_map(|event| match event {
RunObservation::DecisionCall(call) => call.usage,
_ => None,
@@ -208,7 +215,7 @@ async fn direct_mode_serves_the_selected_group() -> Result<(), Box<dyn std::erro
None,
)
.await?;
- assert_eq!(output.selected_model, "premium/model");
+ assert_eq!(output.selected_model, "premium");
let requests = server.received_requests().await.unwrap();
let decision: Value = serde_json::from_slice(&requests[0].body)?;
let context: Value = serde_json::from_str(decision["input"].as_str().unwrap())?;
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index 2c0a87c91..e964e8d0e 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -152,6 +152,8 @@ struct DecisionTargetResponse {
model: ModelId,
llm_client: DecisionLlmClientResponse,
extra_body: BTreeMap<String, Value>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ reasoning_effort: Option<String>,
}
/// Non-secret client settings needed to call a selected model.
@@ -191,11 +193,16 @@ impl SharedRoutingLog {
fn append(
&self,
context: routing_log::RoutingLogContext,
+ target: &str,
model: &str,
tier: Option<&str>,
usage: &Usage,
) {
- if let Err(error) = self.writer.lock().append(context, model, tier, usage) {
+ if let Err(error) = self
+ .writer
+ .lock()
+ .append(context, target, model, tier, usage)
+ {
tracing::warn!(path = %self.path.display(), %error, "routing log append failed");
}
}
@@ -295,6 +302,7 @@ impl ServerState {
base_url: target.base_url,
},
extra_body: target.extra_body,
+ reasoning_effort: target.reasoning_effort,
};
Some(DecisionResponse {
selected: convert(description.selected),
@@ -464,6 +472,7 @@ const CLASSIFIER_TIER: &str = "classifier";
fn stats_observer(
stats: StatsAccumulator,
classifier_log: Option<(SharedRoutingLog, routing_log::RoutingLogContext)>,
+ route: Route,
) -> RunObserver {
Arc::new(move |observation| match observation {
RunObservation::Outcome(metadata) => {
@@ -478,8 +487,13 @@ fn stats_observer(
} else {
if let Some((log, context)) = classifier_log.as_ref() {
log.append(
- context.clone(),
+ context.clone().with_reasoning_effort(
+ route
+ .target(&call.selected_model)
+ .and_then(|target| target.reasoning_effort.as_deref()),
+ ),
&call.selected_model,
+ route.provider_model(&call.selected_model),
None,
&Usage::default(),
);
@@ -490,9 +504,15 @@ fn stats_observer(
RunObservation::LlmCall(call) | RunObservation::DecisionCall(call) => {
let latency_ms = call.duration.as_secs_f64() * 1_000.0;
if let Some((log, context)) = classifier_log.as_ref() {
+ let model = route.provider_model(&call.selected_model);
log.append(
- context.clone(),
+ context.clone().with_reasoning_effort(
+ route
+ .target(&call.selected_model)
+ .and_then(|target| target.reasoning_effort.as_deref()),
+ ),
&call.selected_model,
+ model,
Some(CLASSIFIER_TIER),
call.usage.as_ref().unwrap_or(&Usage::default()),
);
@@ -799,7 +819,10 @@ async fn decision_inner(
match observed_translation::encode_response(
&aggregate,
input_format,
- outcome.selected_model_id().ok().map(ModelId::as_str),
+ outcome
+ .selected_model_id()
+ .ok()
+ .map(|model| route.provider_model(model).as_str()),
&ProviderExtensions::default(),
) {
Ok(response) => Some(response),
@@ -1104,9 +1127,11 @@ async fn handle_llm_request(
});
// Only the Codex namespace mapping is needed downstream, not the whole request.
let request_extensions = request.llm_request.extensions.clone();
+ let stats = state.stats.for_route(route.clone());
let observer = stats_observer(
- state.stats.clone(),
+ stats.clone(),
state.routing_log.clone().zip(routing_log_context.clone()),
+ route.clone(),
);
let output = match route.execute(request, Some(observer)).await {
@@ -1119,7 +1144,7 @@ async fn handle_llm_request(
| LlmClientError::ResponseStateConflict,
}) = &error
{
- state.stats.record_response_error(target);
+ stats.record_response_error(target);
}
return runner_error(error);
}
@@ -1131,16 +1156,28 @@ async fn handle_llm_request(
// The response carries the candidate that actually served it. Fall back to the routing
// selection for algorithms that return a response without an offloaded model call.
let served_model = response.served_model().cloned().or(Some(selected_model));
+ // Keep the provider identity separate from the selected target for reporting.
+ let provider_model = served_model
+ .as_ref()
+ .map(|model| route.provider_model(model).to_string());
let mut response = if let Some(served_model) = served_model.as_ref() {
let cache_eligible = cache_probe
.as_ref()
- .map(|probe| state.stats.prefix_eligibility(served_model, probe))
+ .map(|probe| stats.prefix_eligibility(served_model, probe))
.unwrap_or(0.0);
+ let routing_log_context = routing_log_context.map(|context| {
+ context.with_reasoning_effort(
+ route
+ .target(served_model)
+ .and_then(|target| target.reasoning_effort.as_deref()),
+ )
+ });
usage_metrics::observe(
response,
served_model.as_str(),
+ provider_model.as_deref().unwrap_or(served_model.as_str()),
started.0,
- state.stats,
+ stats,
cache_eligible,
state.routing_log.zip(routing_log_context),
)
@@ -1149,7 +1186,7 @@ async fn handle_llm_request(
};
let upstream_headers = std::mem::take(&mut response.upstream_headers);
- let response_model = served_model.as_ref().map(ToString::to_string);
+ let response_model = provider_model.clone();
let mut response = match into_http_response(
response,
wire_format,
@@ -1169,8 +1206,8 @@ async fn handle_llm_request(
}
response_headers.append(name.clone(), value.clone());
}
- if let Some(served_model) = served_model.as_ref() {
- attach_routing_headers(&mut response, served_model.as_str());
+ if let Some(provider_model) = provider_model.as_ref() {
+ attach_routing_headers(&mut response, provider_model);
}
response
}
@@ -1858,7 +1895,26 @@ targets = ["only"]
headers.insert("x-switchyard-turn-id", "turn-1".parse()?);
let metadata = metadata_from_headers(headers);
let context = routing_log::RoutingLogContext::from_metadata(&metadata);
- let observer = stats_observer(StatsAccumulator::default(), Some((log.clone(), context)));
+ let runner = Runner::from_toml(
+ r#"
+schema_version = 1
+[llm_clients.upstream]
+format = "openai_chat"
+base_url = "http://localhost:1/v1"
+[targets.judge]
+id = "judge-model"
+llm_client = "upstream"
+[routes.only]
+id = "route"
+type = "passthrough"
+target = "judge"
+"#,
+ )?;
+ let observer = stats_observer(
+ StatsAccumulator::default(),
+ Some((log.clone(), context)),
+ runner.route("route").expect("configured route").clone(),
+ );
let call = |model: &str, answer: bool| {
let observation = ModelCallObservation {
@@ -1877,7 +1933,7 @@ targets = ["only"]
RunObservation::LlmCall(observation)
}
};
- observer(call("judge-model", false));
+ observer(call("judge", false));
observer(call("routed-model", true));
observer(RunObservation::Outcome(libsy::OutcomeMetadata::new(
"stage".into(),
@@ -1923,6 +1979,8 @@ targets = ["only"]
serde_json::from_str(contents.lines().next().expect("call record"))?;
assert_eq!(record["session_id"], "session-1");
assert_eq!(record["turn_id"], "turn-1");
+ assert_eq!(record["target"], "judge");
+ assert_eq!(record["model"], "judge-model");
Ok(())
}
diff --git a/crates/switchyard-server/src/routing_log.rs b/crates/switchyard-server/src/routing_log.rs
index 388005175..b6f8391a9 100644
--- a/crates/switchyard-server/src/routing_log.rs
+++ b/crates/switchyard-server/src/routing_log.rs
@@ -54,6 +54,7 @@ impl RoutingLog {
pub(crate) fn append(
&mut self,
context: RoutingLogContext,
+ target: &str,
model: &str,
tier: Option<&str>,
usage: &Usage,
@@ -83,6 +84,8 @@ impl RoutingLog {
agent_role: context.agent.role.map(Cow::Owned),
subagent_identity_unsupported: context.agent.identity_unsupported,
model: model.into(),
+ target: target.into(),
+ reasoning_effort: context.reasoning_effort.map(Cow::Owned),
tier: tier.unwrap_or("").into(),
prompt_tokens: usage.prompt_tokens,
cached_tokens: usage.cached_tokens,
@@ -191,6 +194,7 @@ pub(crate) struct RoutingLogContext {
session_id: Option<String>,
turn_id: Option<String>,
agent: AgentIdentity,
+ reasoning_effort: Option<String>,
}
impl RoutingLogContext {
@@ -216,9 +220,16 @@ impl RoutingLogContext {
.map(str::to_string)
}),
agent: AgentIdentity::from_metadata(metadata),
+ reasoning_effort: None,
}
}
+ /// Captures configured effort for this particular target's call.
+ pub(crate) fn with_reasoning_effort(mut self, effort: Option<&str>) -> Self {
+ self.reasoning_effort = effort.map(str::to_owned);
+ self
+ }
+
/// Attaches the resolved route identity shared by every log entry for this request.
pub(crate) fn with_route(mut self, route_id: impl Into<String>, algorithm: &str) -> Self {
self.route_id = route_id.into();
@@ -264,7 +275,13 @@ struct RoutingRecord<'a> {
agent_role: Option<Cow<'a, str>>,
#[serde(skip_serializing_if = "is_false")]
subagent_identity_unsupported: bool,
+ /// Provider model that served the call, which is what cost tools price.
model: Cow<'a, str>,
+ /// Configured target that served the call. Targets may share a model.
+ #[serde(default, borrow)]
+ target: Cow<'a, str>,
+ #[serde(borrow, skip_serializing_if = "Option::is_none")]
+ reasoning_effort: Option<Cow<'a, str>>,
tier: Cow<'a, str>,
prompt_tokens: u64,
cached_tokens: u64,
@@ -294,6 +311,7 @@ pub(crate) struct SessionStatsSnapshot {
total_cache_creation_tokens: u64,
total_completion_tokens: u64,
models: BTreeMap<ModelId, SessionModelStats>,
+ targets: BTreeMap<ModelId, SessionTargetStats>,
#[serde(skip_serializing_if = "Vec::is_empty")]
outcomes: Vec<LoggedOutcome>,
}
@@ -321,6 +339,29 @@ struct SessionModelStats {
completion_tokens: u64,
}
+#[derive(Serialize)]
+struct SessionTargetStats {
+ model: String,
+ reasoning_effort: Option<String>,
+ #[serde(flatten)]
+ stats: SessionModelStats,
+}
+
+impl SessionModelStats {
+ fn add_record(&mut self, record: &RoutingRecord<'_>) {
+ self.cost.add(record.cost);
+ self.calls = self.calls.saturating_add(1);
+ self.prompt_tokens = self.prompt_tokens.saturating_add(record.prompt_tokens);
+ self.cached_tokens = self.cached_tokens.saturating_add(record.cached_tokens);
+ self.cache_creation_tokens = self
+ .cache_creation_tokens
+ .saturating_add(record.cache_creation_tokens);
+ self.completion_tokens = self
+ .completion_tokens
+ .saturating_add(record.completion_tokens);
+ }
+}
+
impl SessionStatsSnapshot {
fn add_record(&mut self, mut record: RoutingRecord<'_>, session_id: &str) {
if record.session_id.as_deref() != Some(session_id) {
@@ -335,22 +376,25 @@ impl SessionStatsSnapshot {
model => model,
};
let stats = self.models.entry(ModelId::from(model)).or_default();
- stats.cost.add(record.cost);
+ stats.add_record(&record);
+ // Older logs have no target identity; keep them only in the model aggregate.
+ if !record.target.is_empty() {
+ self.targets
+ .entry(ModelId::from(record.target.as_ref()))
+ .or_insert_with(|| SessionTargetStats {
+ model: model.to_string(),
+ reasoning_effort: record.reasoning_effort.as_deref().map(str::to_owned),
+ stats: SessionModelStats::default(),
+ })
+ .stats
+ .add_record(&record);
+ }
self.cost.add(record.cost);
if record.tier == "classifier" {
self.routing_cost.add(record.cost);
} else {
self.answer_cost.add(record.cost);
}
- stats.calls = stats.calls.saturating_add(1);
- stats.prompt_tokens = stats.prompt_tokens.saturating_add(record.prompt_tokens);
- stats.cached_tokens = stats.cached_tokens.saturating_add(record.cached_tokens);
- stats.cache_creation_tokens = stats
- .cache_creation_tokens
- .saturating_add(record.cache_creation_tokens);
- stats.completion_tokens = stats
- .completion_tokens
- .saturating_add(record.completion_tokens);
}
/// Session totals are exactly the sum of the per-model stats, so they are
@@ -441,6 +485,7 @@ mod tests {
let mut log = RoutingLog::new(&path).expect("routing log");
log.append(
RoutingLogContext::from_metadata(&metadata).with_route("model-router", "passthrough"),
+ "worker",
"worker-model",
None,
&Usage::default(),
@@ -484,6 +529,7 @@ mod tests {
assert_eq!(stats.total_prompt_tokens, 15);
assert_eq!(stats.total_completion_tokens, 2);
assert_eq!(stats.models["m1"].calls, 1);
+ assert!(stats.targets.is_empty());
assert_eq!(stats.models["unknown"].prompt_tokens, 5);
assert!(snapshot(&path, "missing").expect("read log").is_none());
}
diff --git a/crates/switchyard-server/src/stats/accumulator.rs b/crates/switchyard-server/src/stats/accumulator.rs
index 357b342eb..3f1ac0208 100644
--- a/crates/switchyard-server/src/stats/accumulator.rs
+++ b/crates/switchyard-server/src/stats/accumulator.rs
@@ -13,6 +13,7 @@ use serde::Serialize;
use super::algorithms::{AlgorithmStats, AlgorithmStatsSnapshot};
use super::cache_eligibility::PrefixProbe;
use switchyard_protocol::ModelId;
+use switchyard_runner::{DecisionTarget, Route};
const MAX_LATENCY_SAMPLES: usize = 10_000;
@@ -31,6 +32,7 @@ pub(crate) struct TokenUsage {
#[derive(Clone)]
pub(crate) struct StatsAccumulator {
inner: Arc<Mutex<StatsAccumulatorInner>>,
+ route: Option<Route>,
}
impl Default for StatsAccumulator {
@@ -47,16 +49,34 @@ impl StatsAccumulator {
) -> Self {
Self {
inner: Arc::new(Mutex::new(StatsAccumulatorInner::new(registry, algorithms))),
+ route: None,
}
}
+ /// Shares counters while resolving this route's target names to provider models.
+ pub(crate) fn for_route(&self, route: Route) -> Self {
+ Self {
+ inner: Arc::clone(&self.inner),
+ route: Some(route),
+ }
+ }
+
+ /// Resolves a stats key to the model used by existing metric labels.
+ pub(crate) fn provider_model(&self, target: &ModelId) -> ModelId {
+ self.route
+ .as_ref()
+ .map_or(target, |route| route.provider_model(target))
+ .clone()
+ }
+
/// Records one successful routed backend call.
pub(crate) fn record_success(&self, model: impl Into<ModelId>, backend_latency_ms: f64) {
let mut inner = self.lock();
inner.total_requests = inner.total_requests.saturating_add(1);
- let stats = inner.model_stats_mut(model.into());
- stats.calls = stats.calls.saturating_add(1);
- stats.model_call_latency.record(backend_latency_ms);
+ for stats in inner.model_stats_mut(model.into(), self.route.as_ref()) {
+ stats.calls = stats.calls.saturating_add(1);
+ stats.model_call_latency.record(backend_latency_ms);
+ }
}
/// Records one failed routed backend call.
@@ -64,8 +84,9 @@ impl StatsAccumulator {
let mut inner = self.lock();
inner.total_requests = inner.total_requests.saturating_add(1);
inner.total_errors = inner.total_errors.saturating_add(1);
- let stats = inner.model_stats_mut(model.into());
- stats.errors = stats.errors.saturating_add(1);
+ for stats in inner.model_stats_mut(model.into(), self.route.as_ref()) {
+ stats.errors = stats.errors.saturating_add(1);
+ }
}
/// Record a failure while processing a buffered response or stream after its routed call
@@ -73,8 +94,9 @@ impl StatsAccumulator {
pub(crate) fn record_response_error(&self, model: impl Into<ModelId>) {
let mut inner = self.lock();
inner.total_errors = inner.total_errors.saturating_add(1);
- let stats = inner.model_stats_mut(model.into());
- stats.errors = stats.errors.saturating_add(1);
+ for stats in inner.model_stats_mut(model.into(), self.route.as_ref()) {
+ stats.errors = stats.errors.saturating_add(1);
+ }
}
/// Records usage and terminal latency after a successful routed call.
@@ -85,9 +107,10 @@ impl StatsAccumulator {
total_latency_ms: f64,
) {
let mut inner = self.lock();
- let stats = inner.model_stats_mut(model.into());
- stats.add_usage(usage);
- stats.total_latency.record(total_latency_ms);
+ for stats in inner.model_stats_mut(model.into(), self.route.as_ref()) {
+ stats.add_usage(usage);
+ stats.total_latency.record(total_latency_ms);
+ }
}
/// Records routing time for one completed algorithm run.
@@ -104,13 +127,14 @@ impl StatsAccumulator {
) {
let mut inner = self.lock();
inner.classifier_requests = inner.classifier_requests.saturating_add(1);
- let stats = inner.classifier_stats_mut(model.into());
- stats.calls = stats.calls.saturating_add(1);
- if let Some(usage) = usage {
- stats.add_usage(usage);
+ for stats in inner.classifier_stats_mut(model.into(), self.route.as_ref()) {
+ stats.calls = stats.calls.saturating_add(1);
+ if let Some(usage) = usage {
+ stats.add_usage(usage);
+ }
+ stats.model_call_latency.record(latency_ms);
+ stats.total_latency.record(latency_ms);
}
- stats.model_call_latency.record(latency_ms);
- stats.total_latency.record(latency_ms);
}
/// Records one failed classifier or judge call.
@@ -118,19 +142,24 @@ impl StatsAccumulator {
let mut inner = self.lock();
inner.classifier_requests = inner.classifier_requests.saturating_add(1);
inner.classifier_errors = inner.classifier_errors.saturating_add(1);
- let stats = inner.classifier_stats_mut(model.into());
- stats.errors = stats.errors.saturating_add(1);
+ for stats in inner.classifier_stats_mut(model.into(), self.route.as_ref()) {
+ stats.errors = stats.errors.saturating_add(1);
+ }
}
/// Returns the cache-eligible fraction for `model` and records the prefix as seen.
pub(crate) fn prefix_eligibility(&self, model: &ModelId, probe: &PrefixProbe) -> f64 {
let mut inner = self.lock();
- let stats = inner.model_stats_mut(model.clone());
- let fraction = probe.eligible_fraction(&stats.seen_prefixes);
- if let Some(hash) = probe.full_hash() {
- stats.seen_prefixes.insert(hash);
- }
- fraction
+ // Cache eligibility stays target-specific when prompts or clients differ.
+ inner
+ .model_stats_mut(model.clone(), self.route.as_ref())
+ .fold(0.0, |_, stats| {
+ let fraction = probe.eligible_fraction(&stats.seen_prefixes);
+ if let Some(hash) = probe.full_hash() {
+ stats.seen_prefixes.insert(hash);
+ }
+ fraction
+ })
}
/// Returns a serializable point-in-time snapshot.
@@ -150,11 +179,13 @@ impl StatsAccumulator {
struct StatsAccumulatorInner {
by_model: BTreeMap<ModelId, ModelStats>,
+ by_target: BTreeMap<ModelId, TargetStats>,
total_requests: u64,
total_errors: u64,
routing_overhead: LatencyHistogram,
routing_fallbacks: RoutingFallbackStats,
by_classifier: BTreeMap<ModelId, ModelStats>,
+ by_classifier_target: BTreeMap<ModelId, TargetStats>,
classifier_requests: u64,
classifier_errors: u64,
algorithm_stats: AlgorithmStats,
@@ -164,33 +195,52 @@ impl StatsAccumulatorInner {
fn new<'a>(registry: Registry, algorithms: impl IntoIterator<Item = &'a str>) -> Self {
Self {
by_model: BTreeMap::new(),
+ by_target: BTreeMap::new(),
total_requests: 0,
total_errors: 0,
routing_overhead: LatencyHistogram::default(),
routing_fallbacks: RoutingFallbackStats::default(),
by_classifier: BTreeMap::new(),
+ by_classifier_target: BTreeMap::new(),
classifier_requests: 0,
classifier_errors: 0,
algorithm_stats: AlgorithmStats::new(registry, algorithms),
}
}
- fn model_stats_mut(&mut self, model: ModelId) -> &mut ModelStats {
- self.by_model.entry(model).or_default()
+ fn model_stats_mut(
+ &mut self,
+ selected: ModelId,
+ route: Option<&Route>,
+ ) -> impl Iterator<Item = &mut ModelStats> {
+ stats_entries(&mut self.by_model, &mut self.by_target, selected, route)
}
- fn classifier_stats_mut(&mut self, model: ModelId) -> &mut ModelStats {
- self.by_classifier.entry(model).or_default()
+ fn classifier_stats_mut(
+ &mut self,
+ selected: ModelId,
+ route: Option<&Route>,
+ ) -> impl Iterator<Item = &mut ModelStats> {
+ stats_entries(
+ &mut self.by_classifier,
+ &mut self.by_classifier_target,
+ selected,
+ route,
+ )
}
fn snapshot(&self) -> StatsSnapshot {
- let (models, total_tokens) = build_model_snapshots(&self.by_model, self.total_requests);
- let classifier = build_classifier_snapshot(
+ let (models, total_tokens) =
+ build_model_snapshots(self.by_model.iter(), self.total_requests);
+ let mut classifier = build_classifier_snapshot(
&self.by_classifier,
self.classifier_requests,
self.classifier_errors,
);
+ classifier.targets =
+ build_target_snapshots(&self.by_classifier_target, self.classifier_requests);
StatsSnapshot {
+ targets: build_target_snapshots(&self.by_target, self.total_requests),
total_requests: self.total_requests,
total_errors: self.total_errors,
total_tokens,
@@ -204,17 +254,81 @@ impl StatsAccumulatorInner {
fn reset(&mut self) {
self.by_model.clear();
+ self.by_target.clear();
self.total_requests = 0;
self.total_errors = 0;
self.routing_overhead = LatencyHistogram::default();
self.routing_fallbacks = RoutingFallbackStats::default();
self.by_classifier.clear();
+ self.by_classifier_target.clear();
self.classifier_requests = 0;
self.classifier_errors = 0;
self.algorithm_stats.reset();
}
}
+/// One configured target's counters and the provider settings that distinguish it.
+struct TargetStats {
+ model: ModelId,
+ reasoning_effort: Option<String>,
+ stats: ModelStats,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize)]
+pub(crate) struct TargetStatsSnapshot {
+ model: ModelId,
+ reasoning_effort: Option<String>,
+ #[serde(flatten)]
+ stats: ModelStatsSnapshot,
+}
+
+/// Updates the legacy model aggregate and the independent target breakdown together.
+fn stats_entries<'a>(
+ models: &'a mut BTreeMap<ModelId, ModelStats>,
+ targets: &'a mut BTreeMap<ModelId, TargetStats>,
+ selected: ModelId,
+ route: Option<&Route>,
+) -> impl Iterator<Item = &'a mut ModelStats> + use<'a> {
+ let target = route.and_then(|route| route.target(&selected));
+ let model = target.map_or(&selected, |target| &target.model).clone();
+ let model_stats = models.entry(model).or_default();
+ let target_stats = target.map(|target: &DecisionTarget| {
+ &mut targets
+ .entry(selected)
+ .or_insert_with(|| TargetStats {
+ model: target.model.clone(),
+ reasoning_effort: target.reasoning_effort.clone(),
+ stats: ModelStats::default(),
+ })
+ .stats
+ });
+ std::iter::once(model_stats).chain(target_stats)
+}
+
+fn build_target_snapshots(
+ targets: &BTreeMap<ModelId, TargetStats>,
+ requests: u64,
+) -> BTreeMap<ModelId, TargetStatsSnapshot> {
+ build_model_snapshots(
+ targets.iter().map(|(name, target)| (name, &target.stats)),
+ requests,
+ )
+ .0
+ .into_iter()
+ .map(|(name, stats)| {
+ let target = &targets[&name];
+ (
+ name,
+ TargetStatsSnapshot {
+ model: target.model.clone(),
+ reasoning_effort: target.reasoning_effort.clone(),
+ stats,
+ },
+ )
+ })
+ .collect()
+}
+
#[derive(Clone, Debug, Default)]
struct ModelStats {
calls: u64,
@@ -318,6 +432,7 @@ pub(crate) struct StatsSnapshot {
pub total_errors: u64,
pub total_tokens: TokenTotals,
pub models: BTreeMap<ModelId, ModelStatsSnapshot>,
+ pub targets: BTreeMap<ModelId, TargetStatsSnapshot>,
pub routing_overhead: LatencyHistogramSnapshot,
pub routing_fallbacks: RoutingFallbackStats,
pub classifier: ClassifierStatsSnapshot,
@@ -339,6 +454,7 @@ pub(crate) struct ClassifierStatsSnapshot {
pub total_errors: u64,
pub total_tokens: TokenTotals,
pub models: BTreeMap<ModelId, ModelStatsSnapshot>,
+ pub targets: BTreeMap<ModelId, TargetStatsSnapshot>,
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize)]
@@ -383,12 +499,12 @@ pub(crate) struct LatencyHistogramSnapshot {
pub p99_ms: f64,
}
-fn build_model_snapshots(
- by_model: &BTreeMap<ModelId, ModelStats>,
+fn build_model_snapshots<'a>(
+ by_model: impl Iterator<Item = (&'a ModelId, &'a ModelStats)> + Clone,
total_requests: u64,
) -> (BTreeMap<ModelId, ModelStatsSnapshot>, TokenTotals) {
let mut totals = TokenTotals::default();
- for stats in by_model.values() {
+ for (_, stats) in by_model.clone() {
totals.prompt = totals.prompt.saturating_add(stats.prompt_tokens);
totals.completion = totals.completion.saturating_add(stats.completion_tokens);
totals.cached = totals.cached.saturating_add(stats.cached_tokens);
@@ -400,7 +516,6 @@ fn build_model_snapshots(
totals.total = totals.prompt.saturating_add(totals.completion);
let models = by_model
- .iter()
.map(|(model, stats)| {
let total_tokens = stats.prompt_tokens.saturating_add(stats.completion_tokens);
let snapshot = ModelStatsSnapshot {
@@ -436,12 +551,13 @@ fn build_classifier_snapshot(
total_requests: u64,
total_errors: u64,
) -> ClassifierStatsSnapshot {
- let (models, total_tokens) = build_model_snapshots(models, total_requests);
+ let (models, total_tokens) = build_model_snapshots(models.iter(), total_requests);
ClassifierStatsSnapshot {
total_requests,
total_errors,
total_tokens,
models,
+ targets: BTreeMap::new(),
}
}
@@ -492,6 +608,67 @@ mod tests {
}
}
+ /// Shared-model targets contribute once to totals, with independent errors and cache state.
+ #[test]
+ fn shared_model_targets_keep_independent_counters_and_reset_together() {
+ let runner = switchyard_runner::Runner::from_toml(
+ r#"
+schema_version = 1
+[llm_clients.upstream]
+format = "openai_chat"
+base_url = "http://localhost:1/v1"
+[targets.efficient]
+id = "shared"
+llm_client = "upstream"
+reasoning_effort = "medium"
+[targets.capable]
+id = "shared"
+llm_client = "upstream"
+reasoning_effort = "max"
+[routes.test]
+id = "route"
+type = "random"
+targets = ["efficient", "capable"]
+"#,
+ )
+ .unwrap();
+ let root = StatsAccumulator::default();
+ let stats = root.for_route(runner.route("route").unwrap().clone());
+ for target in ["efficient", "capable"] {
+ stats.record_success(target, 10.0);
+ stats.record_usage(target, usage(10, 2), 15.0);
+ stats.record_classifier_success(target, Some(usage(5, 1)), 3.0);
+ }
+ stats.record_error("efficient");
+ stats.record_response_error("capable");
+ stats.record_classifier_error("capable");
+ let snapshot = root.snapshot();
+ assert_eq!(snapshot.total_requests, 3);
+ assert_eq!(snapshot.total_errors, 2);
+ assert_eq!(snapshot.total_tokens.prompt, 20);
+ assert_eq!(snapshot.models["shared"].calls, 2);
+ assert_eq!(snapshot.models["shared"].errors, 2);
+ assert_eq!(snapshot.models["shared"].model_call_latency.count, 2);
+ assert_eq!(snapshot.targets["efficient"].stats.calls, 1);
+ assert_eq!(snapshot.targets["efficient"].stats.errors, 1);
+ assert_eq!(snapshot.targets["capable"].stats.errors, 1);
+ assert_eq!(snapshot.classifier.total_requests, 3);
+ assert_eq!(snapshot.classifier.models["shared"].calls, 2);
+ assert_eq!(snapshot.classifier.total_tokens.prompt, 10);
+ assert_eq!(snapshot.classifier.targets["capable"].stats.errors, 1);
+ assert_eq!(
+ snapshot.targets["capable"].reasoning_effort.as_deref(),
+ Some("max")
+ );
+ let probe = prefix_probe(&json!({"messages": [{"role": "user", "content": "repeated"}]}));
+ assert_eq!(stats.prefix_eligibility(&"efficient".into(), &probe), 0.0);
+ assert_eq!(stats.prefix_eligibility(&"capable".into(), &probe), 0.0);
+ assert_eq!(stats.prefix_eligibility(&"efficient".into(), &probe), 1.0);
+ root.reset();
+ assert_eq!(root.snapshot(), StatsSnapshot::default());
+ assert_eq!(stats.prefix_eligibility(&"efficient".into(), &probe), 0.0);
+ }
+
#[test]
fn snapshot_aggregates_backend_and_classifier_stats() {
let stats = StatsAccumulator::default();
diff --git a/crates/switchyard-server/src/usage_metrics.rs b/crates/switchyard-server/src/usage_metrics.rs
index 54c2a1c01..f7d9a331a 100644
--- a/crates/switchyard-server/src/usage_metrics.rs
+++ b/crates/switchyard-server/src/usage_metrics.rs
@@ -14,9 +14,11 @@ use crate::routing_log::RoutingLogContext;
use crate::stats::{StatsAccumulator, TokenUsage};
/// Observes a routed response without changing its aggregate or streaming contents.
+/// `model` is the served target; stats retain both target and provider aggregates.
pub(crate) fn observe(
response: Response,
model: &str,
+ provider_model: &str,
started: Instant,
stats: StatsAccumulator,
cache_eligible: f64,
@@ -28,12 +30,13 @@ pub(crate) fn observe(
upstream_headers,
} = response;
let model = model.to_string();
+ let provider_model = provider_model.to_string();
let llm_response = match llm_response {
LlmResponse::Agg(agg) => {
record_terminal(&stats, &agg.usage, &model, started, cache_eligible);
if let Some((log, context)) = routing_log {
- log.append(context, &model, None, &agg.usage);
+ log.append(context, &model, &provider_model, None, &agg.usage);
}
LlmResponse::Agg(agg)
}
@@ -68,7 +71,7 @@ pub(crate) fn observe(
}
if failed && !recorded
&& let Some((log, context)) = routing_log.as_ref() {
- log.append(context.clone(), &model, None, &Usage::default());
+ log.append(context.clone(), &model, &provider_model, None, &Usage::default());
}
if failed {
record_stream_error(&stats, &model);
@@ -80,7 +83,7 @@ pub(crate) fn observe(
{
record_terminal(&stats, usage, &model, started, cache_eligible);
if let Some((log, context)) = routing_log.as_ref() {
- log.append(context.clone(), &model, None, usage);
+ log.append(context.clone(), &model, &provider_model, None, usage);
}
recorded = true;
}
@@ -93,7 +96,7 @@ pub(crate) fn observe(
let usage = latest_usage.unwrap_or_default();
record_terminal(&stats, &usage, &model, started, cache_eligible);
if let Some((log, context)) = routing_log {
- log.append(context, &model, None, &usage);
+ log.append(context, &model, &provider_model, None, &usage);
}
}
};
@@ -139,8 +142,9 @@ fn record_terminal(
cache_eligible: f64,
) {
let total_latency = started.elapsed();
- record_usage(usage, model);
- record_latency(model, total_latency);
+ let provider_model = stats.provider_model(&model.into());
+ record_usage(usage, &provider_model);
+ record_latency(&provider_model, total_latency);
let mut token_usage = token_usage(usage);
token_usage.cacheable_prompt_tokens =
(token_usage.prompt_tokens as f64 * cache_eligible).round() as u64;
@@ -221,6 +225,7 @@ mod tests {
let observed = observe(
response,
"model/worker",
+ "model/worker",
Instant::now(),
stats.clone(),
0.0,
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 5fb4c6f24..5adb4c20a 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -831,6 +831,7 @@ fn random_state_with_retries(
format: WireFormat::OpenAiChat,
base_url: base_url.to_string(),
extra_body: BTreeMap::new(),
+ reasoning_effort: None,
})
.collect();
Ok((
@@ -922,6 +923,7 @@ async fn stats_exposes_the_exact_empty_schema_and_no_legacy_alias() -> TestResul
"total_errors": 0,
"total_tokens": empty_token_totals(),
"models": {},
+ "targets": {},
"routing_overhead": {
"count": 0,
"total_ms": 0.0,
@@ -940,6 +942,7 @@ async fn stats_exposes_the_exact_empty_schema_and_no_legacy_alias() -> TestResul
"total_errors": 0,
"total_tokens": empty_token_totals(),
"models": {},
+ "targets": {},
},
"algorithm_stats": {},
})
@@ -1186,7 +1189,9 @@ async fn failed_responses_return_errors_and_try_fallback_across_endpoints() -> T
}
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(stats["models"]["model/failed"]["errors"], requests.len());
+ assert_eq!(stats["targets"]["first"]["errors"], requests.len());
assert_eq!(stats["models"]["model/fallback"]["calls"], requests.len());
+ assert_eq!(stats["targets"]["second"]["calls"], requests.len());
Ok(())
}
@@ -2474,7 +2479,7 @@ escalation = {{ confirmations = 1 }}
"algorithm": "llm_task_classifier",
"evidence": {"source": "llm-classifier", "score": 0.9, "threshold": 0.5},
"judgments": [{
- "target": "model/weak",
+ "target": "economy",
"category": "efficient",
"evidence": {"source": "llm-classifier", "score": 0.9, "threshold": 0.5},
}],
@@ -2671,6 +2676,162 @@ reasoning = true
Ok(())
}
+#[tokio::test]
+async fn targets_sharing_a_model_keep_their_own_settings() -> TestResult {
+ let upstream = MockUpstream::start().await?;
+ let state = load_test_config(&format!(
+ r#"
+schema_version = 1
+
+[llm_clients.upstream]
+format = "openai_chat"
+base_url = "{base_url}"
+
+[targets.efficient]
+id = "model/shared"
+llm_client = "upstream"
+reasoning_effort = "medium"
+system_prompt = "efficient prompt"
+
+[targets.capable]
+id = "model/shared"
+llm_client = "upstream"
+reasoning_effort = "max"
+system_prompt = "capable prompt"
+
+[routes.random]
+id = "switchyard/random"
+type = "random"
+targets = ["efficient", "capable"]
+seed = 17
+"#,
+ base_url = upstream.base_url
+ ))?;
+ let log_dir = tempfile::tempdir()?;
+ let log_path = log_dir.path().join("routing.jsonl");
+ let app = build_switchyard_router(state.with_routing_log(&log_path)?);
+
+ let requests = 16;
+ for index in 0..requests {
+ let streaming = index % 2 == 0;
+ let response = send_with_headers(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({
+ "model": "switchyard/random",
+ "messages": [{"role": "user", "content": "hello"}],
+ "reasoning_effort": "high",
+ "stream": streaming
+ })),
+ &[("x-switchyard-session-id", "shared-efforts")],
+ )
+ .await?;
+ assert_eq!(response.status, StatusCode::OK);
+ assert_eq!(
+ response.headers["x-model-router-selected-model"],
+ "model/shared"
+ );
+ if streaming {
+ assert!(response.text()?.contains("model/shared"));
+ } else {
+ assert_eq!(response.json()?["model"], "model/shared");
+ }
+ }
+
+ let calls = upstream.calls.lock().await;
+ assert_eq!(calls.len(), requests);
+ for call in calls.iter() {
+ assert_eq!(call["model"], "model/shared");
+ let effort = call["reasoning_effort"].as_str().unwrap_or_default();
+ let expected_prompt = match effort {
+ "medium" => "efficient prompt",
+ "max" => "capable prompt",
+ other => panic!("unexpected effort {other}"),
+ };
+ assert!(has_system_prompt(call, expected_prompt), "{call}");
+ }
+ drop(calls);
+
+ let decision = send(
+ &app,
+ "POST",
+ "/v1/decision",
+ Some(json!({
+ "input_format": "openai_chat",
+ "request": {
+ "model": "switchyard/random",
+ "messages": [{"role": "user", "content": "hello"}]
+ }
+ })),
+ )
+ .await?
+ .json()?;
+ let selected = &decision["selected"];
+ assert_eq!(selected["model"], "model/shared");
+ let expected_effort = match selected["target"].as_str() {
+ Some("capable") => "max",
+ Some("efficient") => "medium",
+ other => panic!("unexpected target {other:?}"),
+ };
+ assert_eq!(selected["reasoning_effort"], expected_effort);
+
+ let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
+ let efficient = stats["targets"]["efficient"]["calls"].as_u64().unwrap_or(0);
+ let capable = stats["targets"]["capable"]["calls"].as_u64().unwrap_or(0);
+ assert!(efficient > 0 && capable > 0, "{}", stats["models"]);
+ assert_eq!(efficient + capable, requests as u64);
+ assert_eq!(stats["models"].as_object().unwrap().len(), 1);
+ assert_eq!(stats["models"]["model/shared"]["calls"], requests);
+ for (target, effort) in [("capable", "max"), ("efficient", "medium")] {
+ assert_eq!(stats["targets"][target]["model"], "model/shared");
+ assert_eq!(stats["targets"][target]["reasoning_effort"], effort);
+ }
+ assert_eq!(
+ stats["models"]["model/shared"]["prompt_tokens"],
+ stats["total_tokens"]["prompt"]
+ );
+ let session = send(
+ &app,
+ "GET",
+ "/v1/routing/session-stats?session_id=shared-efforts",
+ None,
+ )
+ .await?
+ .json()?;
+ assert_eq!(session["total_calls"], requests);
+ assert_eq!(session["models"]["model/shared"]["calls"], requests);
+ for target in ["capable", "efficient"] {
+ assert_eq!(
+ session["targets"][target]["calls"],
+ stats["targets"][target]["calls"]
+ );
+ assert_eq!(
+ session["targets"][target]["reasoning_effort"],
+ stats["targets"][target]["reasoning_effort"]
+ );
+ assert_eq!(
+ session["targets"][target]["prompt_tokens"],
+ stats["targets"][target]["prompt_tokens"]
+ );
+ }
+ for line in std::fs::read_to_string(log_path)?.lines() {
+ let record: Value = serde_json::from_str(line)?;
+ assert_eq!(record["model"], "model/shared");
+ let target = record["target"].as_str().unwrap();
+ assert_eq!(
+ record["reasoning_effort"],
+ stats["targets"][target]["reasoning_effort"]
+ );
+ }
+ send(&app, "POST", "/v1/stats/reset", None).await?;
+ let reset = send(&app, "GET", "/v1/stats", None).await?.json()?;
+ assert_eq!(reset["total_requests"], 0);
+ assert_eq!(reset["models"], json!({}));
+ assert_eq!(reset["targets"], json!({}));
+ Ok(())
+}
+
#[tokio::test]
async fn toml_config_constructs_and_serves_multiple_algorithms() -> TestResult {
let upstream = MockUpstream::start().await?;
@@ -3348,6 +3509,7 @@ escalation = {{ confirmations = 1 }}
assert_eq!(stats["total_calls"], 2);
assert_eq!(stats["total_prompt_tokens"], 20);
assert_eq!(stats["total_completion_tokens"], 4);
+ // Session stats come from the routing log, which prices by provider model.
assert_eq!(stats["models"]["model/weak"]["calls"], 1);
assert_eq!(stats["models"]["model/strong"]["calls"], 1);
@@ -4691,12 +4853,14 @@ format = "openai_chat"
base_url = "{base_url}"
[targets.shared]
-id = "model/shared"
+id = "model/route-sanity"
llm_client = "upstream"
+reasoning_effort = "medium"
[targets.capable]
-id = "model/capable"
+id = "model/route-sanity"
llm_client = "upstream"
+reasoning_effort = "max"
[routes.passthrough]
id = "route/passthrough"
@@ -4716,39 +4880,158 @@ confidence_threshold = 1.0
.with_routing_log(&log_path)?;
let app = build_switchyard_router(state);
- for route_id in ["route/passthrough", "route/stage"] {
- let response = send(
- &app,
- "POST",
- "/v1/chat/completions",
- Some(json!({
- "model": route_id,
- "messages": [{"role": "user", "content": "hello"}]
- })),
- )
- .await?;
- assert_eq!(response.status, StatusCode::OK, "{route_id}");
- assert_eq!(
- response
- .headers
- .get("x-model-router-selected-model")
- .and_then(|value| value.to_str().ok()),
- Some("model/shared"),
- "{route_id}"
- );
+ let before = send(&app, "GET", "/metrics", None).await?;
+ let normal = json!([{"role": "user", "content": "hello"}]);
+ let critical = json!([
+ {"role": "user", "content": "fix the build"},
+ {"role": "assistant", "tool_calls": [{
+ "id": "call_1", "type": "function",
+ "function": {"name": "Bash", "arguments": "{\"command\": \"cargo test\"}"}
+ }]},
+ {"role": "tool", "tool_call_id": "call_1", "content": "fatal runtime error: out of memory"}
+ ]);
+ for (route_id, messages) in [
+ ("route/passthrough", &normal),
+ ("route/stage", &normal),
+ ("route/stage", &critical),
+ ] {
+ for streaming in [false, true] {
+ let response = send_with_headers(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({"model": route_id, "messages": messages, "stream": streaming})),
+ &[("x-switchyard-session-id", "route-sanity")],
+ )
+ .await?;
+ assert_eq!(response.status, StatusCode::OK, "{route_id}");
+ assert_eq!(
+ response.headers["x-model-router-selected-model"],
+ "model/route-sanity"
+ );
+ if streaming {
+ assert!(response.text()?.contains("model/route-sanity"));
+ } else {
+ assert_eq!(response.json()?["model"], "model/route-sanity");
+ }
+ }
}
let records = std::fs::read_to_string(&log_path)?
.lines()
.map(serde_json::from_str::<Value>)
.collect::<Result<Vec<_>, _>>()?;
- assert_eq!(records.len(), 2);
- assert_eq!(records[0]["route_id"], "route/passthrough");
- assert_eq!(records[0]["algorithm"], "passthrough");
- assert_eq!(records[0]["model"], "model/shared");
- assert_eq!(records[1]["route_id"], "route/stage");
- assert_eq!(records[1]["algorithm"], "stage_router");
- assert_eq!(records[1]["model"], "model/shared");
+ assert_eq!(records.len(), 6);
+ for (index, record) in records.iter().enumerate() {
+ let (route, algorithm) = if index < 2 {
+ ("route/passthrough", "passthrough")
+ } else {
+ ("route/stage", "stage_router")
+ };
+ let (target, effort) = if index < 4 {
+ ("shared", "medium")
+ } else {
+ ("capable", "max")
+ };
+ assert_eq!(record["route_id"], route);
+ assert_eq!(record["algorithm"], algorithm);
+ assert_eq!(record["model"], "model/route-sanity");
+ assert_eq!(record["target"], target);
+ assert_eq!(record["reasoning_effort"], effort);
+ assert_eq!(record["session_id"], "route-sanity");
+ }
+ let calls = upstream.calls.lock().await;
+ assert_eq!(calls.len(), 6);
+ for (index, call) in calls.iter().enumerate() {
+ assert_eq!(call["model"], "model/route-sanity");
+ assert_eq!(
+ call["reasoning_effort"],
+ if index < 4 { "medium" } else { "max" }
+ );
+ }
+ drop(calls);
+
+ let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
+ let session = send(
+ &app,
+ "GET",
+ "/v1/routing/session-stats?session_id=route-sanity",
+ None,
+ )
+ .await?
+ .json()?;
+ assert_eq!(stats["total_requests"], 6);
+ assert_eq!(stats["total_errors"], 0);
+ assert_eq!(session["total_calls"], 6);
+ for snapshot in [&stats, &session] {
+ assert_eq!(snapshot["models"].as_object().unwrap().len(), 1);
+ assert_eq!(snapshot["models"]["model/route-sanity"]["calls"], 6);
+ for (target, effort, calls) in [("shared", "medium", 4), ("capable", "max", 2)] {
+ assert_eq!(snapshot["targets"][target]["model"], "model/route-sanity");
+ assert_eq!(snapshot["targets"][target]["reasoning_effort"], effort);
+ assert_eq!(snapshot["targets"][target]["calls"], calls);
+ }
+ }
+ assert_eq!(
+ stats["total_tokens"]["prompt"],
+ session["total_prompt_tokens"]
+ );
+ assert_eq!(
+ stats["total_tokens"]["completion"],
+ session["total_completion_tokens"]
+ );
+ for source in ["override", "capable_hold"] {
+ assert_eq!(
+ stats["algorithm_stats"]["stage_router"]["routing_decisions"][source]["targets"]["model/route-sanity"],
+ 1
+ );
+ }
+
+ let after = send(&app, "GET", "/metrics", None).await?;
+ for (metric, label, expected) in [
+ ("switchyard_requests_total", "model", 6.0),
+ (
+ "switchyard_prompt_tokens_total",
+ "model",
+ session["total_prompt_tokens"].as_f64().unwrap(),
+ ),
+ (
+ "switchyard_completion_tokens_total",
+ "model",
+ session["total_completion_tokens"].as_f64().unwrap(),
+ ),
+ ] {
+ assert_eq!(
+ metric_delta(
+ before.text()?,
+ after.text()?,
+ metric,
+ &[(label, "model/route-sanity")]
+ ),
+ Some(expected),
+ "{metric}"
+ );
+ for target in ["shared", "capable"] {
+ assert!(
+ metric_line(after.text()?, metric, &[(label, target)]).is_none(),
+ "{metric}"
+ );
+ }
+ }
+ for (algorithm, calls) in [("passthrough", 2.0), ("stage_router", 4.0)] {
+ assert_eq!(
+ metric_delta(
+ before.text()?,
+ after.text()?,
+ "switchyard_decisions_total",
+ &[
+ ("algorithm", algorithm),
+ ("selected_model", "model/route-sanity")
+ ],
+ ),
+ Some(calls),
+ );
+ }
Ok(())
}
@@ -4943,7 +5226,9 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted
records
.iter()
.filter(|record| {
- record["model"] == "model/strong" && record.get("fallback_reason").is_none()
+ record["model"] == "model/strong"
+ && record["target"] == "second"
+ && record.get("fallback_reason").is_none()
})
.count(),
3
@@ -5563,6 +5848,7 @@ async fn advisor_route_routing_log_records_classifier_tier() -> TestResult {
.iter()
.find(|record| record["model"] == "model/advisor")
.ok_or("consult row present")?;
+ assert_eq!(consult["target"], "advisor");
assert_eq!(consult["tier"], "classifier");
assert_eq!(consult["route_id"], "switchyard/advisor");
assert_eq!(consult["algorithm"], "advisor_gate");
diff --git a/docs/integrations/nemo_relay.md b/docs/integrations/nemo_relay.md
index ccce9e086..a423ba74f 100644
--- a/docs/integrations/nemo_relay.md
+++ b/docs/integrations/nemo_relay.md
@@ -489,7 +489,11 @@ supported trace projections, but Relay's OTLP logs default to Info. Set
`minimum_severity` to `debug` to include these call records in
[log export](https://docs.nvidia.com/nemo/relay/configure-plugins/observability/opentelemetry#log-export).
-Switchyard marks use the target's upstream model ID, not its local TOML key.
+Switchyard marks keep provider model IDs in `selected_model` and `served_model`.
+Decision marks also include `selected_target`, `served_target`, and the served target's
+configured `reasoning_effort`. Targets sharing one provider model remain distinguishable.
+`fallback_used` compares target identities, so a fallback between two targets sharing a
+provider model is still reported.
Relay request telemetry normally uses the Switchyard route ID, while response
telemetry can report the model that answered.
diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md
index 63f7326f6..370ebea9c 100644
--- a/docs/reference/toml_schema.md
+++ b/docs/reference/toml_schema.md
@@ -170,12 +170,35 @@ such clients.
| `system_prompt` | No | unset | System prompt prepended when this target serves a completion. |
| `extra_body` | No | `{}` | Values merged into the upstream request when the request does not already set that key. |
| `omit_body_fields` | No | `[]` | Top-level fields removed from every request body that Switchyard sends to this target. Switchyard removes them after it translates the request to the LLM client's `format`, so use that format's field names, for example `reasoning_effort` on `openai_chat` or `reasoning` on `openai_responses`. Switchyard applies `extra_body` and `reasoning_effort` after the removal, so either can set a removed field again. |
-| `reasoning_effort` | No | unset | Reasoning effort forced on every request to this target, replacing the value the caller sent (`reasoning.effort` on `openai_responses`, `reasoning_effort` on `openai_chat`). Rejected on `anthropic_messages` clients. Use it to run one target at a different effort than the client asked for, for example a strong tier at `max` behind a client that sends `high`. Targets with different effort settings need distinct model IDs when used within one route. Separate routes may use the same model ID with separate `llm_clients` entries (same endpoint, different name). |
+| `reasoning_effort` | No | unset | Reasoning effort forced on every request to this target, replacing the value the caller sent (`reasoning.effort` on `openai_responses`, `reasoning_effort` on `openai_chat`, `output_config.effort` on `anthropic_messages`, which accepts `low`, `medium`, `high`, `xhigh`, or `max`). Use it to run one target at a different effort than the client asked for, for example a strong tier at `max` behind a client that sends `high`. |
-Within one route, callable targets with the same model ID must use the same `llm_client`.
-This includes completion targets and targets used for judging or classification. Switchyard rejects
-same-model targets on different clients within a route because execution is keyed by model ID.
-Separate routes may use the same model ID on different clients.
+Switchyard routes on target names, so several targets may share one `id`. Each target keeps its
+own `llm_client`, `system_prompt`, `extra_body`, `omit_body_fields`, and `reasoning_effort`. For
+example, one route can run the same model at two efforts:
+
+```toml
+[targets.capable]
+id = "gpt"
+llm_client = "openai"
+reasoning_effort = "max"
+
+[targets.efficient]
+id = "gpt"
+llm_client = "openai"
+reasoning_effort = "medium"
+```
+
+Responses, the `x-model-router-selected-model` header, and the routing log `model` field name the
+target's `id`. Existing model metric labels and the `models` keys in `/v1/stats` and
+`/v1/routing/session-stats` also use provider IDs. Targets sharing a provider model contribute
+to one model aggregate.
+
+Both stats endpoints add a separate `targets` breakdown keyed by TOML target name. Each entry
+includes `model`, configured `reasoning_effort` (or `null`), and that target's counters. The
+process stats also include `classifier.targets` for routing-time calls. Totals count each call
+once; the model and target breakdowns are two views of those calls. Routing log rows retain
+`model` and `target`, and include `reasoning_effort` when configured. Older rows without a
+target still contribute to model totals.
Each selected or fallback target is prepared from the routed request independently. A prompt
configured for one target is therefore not carried into another target's fallback request.
@@ -183,9 +206,8 @@ Judge-only, classifier-only, and reviewer-only targets are not completion destin
receive this prompt.
Escalation's weak target and Advisor's executor produce a candidate response while routing, so
-their target prompt is applied to that call. A prompted target in either role cannot use the same
-model ID as that route's judge or reviewer because those calls would otherwise be indistinguishable
-at the client boundary; Switchyard rejects that configuration when it loads.
+their target prompt is applied to that call. A prompted target in either role cannot also be that
+route's judge or reviewer; Switchyard rejects that configuration when it loads.
Token-count requests do not apply target system prompts.
## `[decision_clients.<name>]`
diff --git a/examples/dynamo-preproc/src/router.rs b/examples/dynamo-preproc/src/router.rs
index 6388b01d5..d6c3d19b9 100644
--- a/examples/dynamo-preproc/src/router.rs
+++ b/examples/dynamo-preproc/src/router.rs
@@ -140,7 +140,7 @@ impl Router {
}
}
ensure!(outcome.response.is_none(), "routing-only contract violated");
- let model = outcome.selected_model_id()?.clone();
+ let model = route.provider_model(outcome.selected_model_id()?).clone();
let mut selected_ir = outcome.request.llm_request;
selected_ir.model = None;
ensure!(
diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py
index d9228e63f..75795e8c3 100644
--- a/switchyard_rust/libsy.py
+++ b/switchyard_rust/libsy.py
@@ -304,7 +304,10 @@ if TYPE_CHECKING:
@final
class ConfiguredAlgorithm:
- """An algorithm and model groups; the host applies target prompts and provider settings."""
+ """LLM calls and outcomes use target names; the host resolves provider IDs and settings.
+
+ Decision calls use provider IDs.
+ """
def run_stream(
self,
diff --git a/tests/test_algorithm_factory.py b/tests/test_algorithm_factory.py
index 62d697090..48d2e95d0 100644
--- a/tests/test_algorithm_factory.py
+++ b/tests/test_algorithm_factory.py
@@ -33,4 +33,4 @@ target = "fast"
]
assert len(steps) == 1
assert isinstance(steps[0], Step.Done)
- assert steps[0].outcome.selected_model_ids == ["gateway/fast"]
+ assert steps[0].outcome.selected_model_ids == ["fast"]