Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(server): record route attribution in durable log (#618)

Clement Pakkam Isaac5879c1097b53
diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md
index 0641f8308..87c281886 100644
--- a/crates/switchyard-server/README.md
+++ b/crates/switchyard-server/README.md
@@ -93,7 +93,9 @@ for equal weighting. The optional `seed` reproduces the selection sequence for t
## Session routing log
Pass `--routing-log-file PATH` to append one JSON record after each completed routed response.
-Streaming responses are recorded after the stream drains. When enabled,
+Streaming responses are recorded after the stream drains. Each record names the client-facing
+`route_id`, routing `algorithm`, served `model`, tier, and token usage so routes sharing a backend
+remain distinguishable. When enabled,
`GET /v1/routing/session-stats?session_id=ID` rescans the durable log and returns call and token
totals for that normalized session ID, normally supplied as `x-switchyard-session-id`, grouped by
served model. The legacy `proxy_x_session_id` remains a fallback when no normalized session ID is
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index 7409f3c12..8b6f06de8 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -1005,6 +1005,7 @@ fn resolve_route(
Ok((route, request))
}
+/// Resolves and executes an LLM request, attaching route identity when durable logging is enabled.
async fn handle_llm_request(
state: ServerState,
started: RequestStart,
@@ -1018,6 +1019,12 @@ async fn handle_llm_request(
Ok(resolved) => resolved,
Err(response) => return response,
};
+ let routing_log_context = routing_log_context.map(|context| {
+ context.with_route(
+ request.llm_request.model.as_deref().unwrap_or_default(),
+ route.algorithm_name(),
+ )
+ });
// Only the Codex namespace mapping is needed downstream, not the whole request.
let request_extensions = request.llm_request.extensions.clone();
let observer = stats_observer(
diff --git a/crates/switchyard-server/src/routing_log.rs b/crates/switchyard-server/src/routing_log.rs
index 47ed65387..2e9dd8637 100644
--- a/crates/switchyard-server/src/routing_log.rs
+++ b/crates/switchyard-server/src/routing_log.rs
@@ -51,6 +51,8 @@ impl RoutingLog {
let usage = token_usage(usage);
let record = RoutingRecord {
ts: format_rfc3339_millis(SystemTime::now()).to_string().into(),
+ route_id: context.route_id.into(),
+ algorithm: context.algorithm.into(),
task: context.task.map(Cow::Owned),
trial_id: context.trial_id.map(Cow::Owned),
session_id: context.session_id.map(Cow::Owned),
@@ -95,9 +97,11 @@ pub(crate) fn snapshot(
Ok((snapshot.total_calls > 0).then_some(snapshot))
}
-/// Request fields retained until terminal usage and routing are available.
+/// Holds request metadata until route resolution attaches the durable route identity.
#[derive(Clone)]
pub(crate) struct RoutingLogContext {
+ route_id: String,
+ algorithm: String,
task: Option<String>,
trial_id: Option<String>,
session_id: Option<String>,
@@ -108,6 +112,8 @@ impl RoutingLogContext {
pub(crate) fn from_metadata(metadata: &Metadata) -> Self {
let headers = metadata.http_headers.as_ref();
Self {
+ route_id: String::new(),
+ algorithm: String::new(),
task: headers
.and_then(|headers| nonempty_header(headers, TASK_HEADER))
.map(str::to_string),
@@ -121,6 +127,13 @@ impl RoutingLogContext {
}),
}
}
+
+ /// Attaches the resolved route identity shared by every log entry for this request.
+ pub(crate) fn with_route(mut self, route_id: impl Into<String>, algorithm: &str) -> Self {
+ self.route_id = route_id.into();
+ self.algorithm = algorithm.to_string();
+ self
+ }
}
/// One appended routing record, and the read schema [`snapshot`] parses back,
@@ -130,6 +143,8 @@ impl RoutingLogContext {
#[serde(default)]
struct RoutingRecord<'a> {
ts: Cow<'a, str>,
+ route_id: Cow<'a, str>,
+ algorithm: Cow<'a, str>,
#[serde(borrow)]
task: Option<Cow<'a, str>>,
#[serde(borrow)]
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 367f94394..43e6abf7f 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -2550,12 +2550,89 @@ async fn routing_log_prefers_canonical_and_preserves_legacy_fallback() -> TestRe
Ok(())
}
+/// Two routes serving one model must remain distinguishable in the durable accounting record.
+#[tokio::test]
+async fn routing_log_attributes_shared_models_to_the_requested_route() -> TestResult {
+ let upstream = MockUpstream::start().await?;
+ let temp_dir = tempfile::tempdir()?;
+ let log_path = temp_dir.path().join("routing.jsonl");
+ let state = load_test_config(&format!(
+ r#"
+schema_version = 1
+
+[llm_clients.upstream]
+format = "openai_chat"
+base_url = "{base_url}"
+
+[targets.shared]
+id = "model/shared"
+llm_client = "upstream"
+
+[targets.capable]
+id = "model/capable"
+llm_client = "upstream"
+
+[routes.passthrough]
+id = "route/passthrough"
+type = "passthrough"
+target = "shared"
+
+[routes.stage]
+id = "route/stage"
+type = "stage_router"
+capable_target = "capable"
+efficient_target = "shared"
+picker = "efficient_first"
+confidence_threshold = 1.0
+"#,
+ base_url = upstream.base_url
+ ))?
+ .with_routing_log(&log_path)?;
+ let app = build_switchyard_router(state);
+
+ for route_id in ["route/passthrough", "route/stage"] {
+ let response = send(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({
+ "model": route_id,
+ "messages": [{"role": "user", "content": "hello"}]
+ })),
+ )
+ .await?;
+ assert_eq!(response.status, StatusCode::OK, "{route_id}");
+ assert_eq!(
+ response
+ .headers
+ .get("x-model-router-selected-model")
+ .and_then(|value| value.to_str().ok()),
+ Some("model/shared"),
+ "{route_id}"
+ );
+ }
+
+ let records = std::fs::read_to_string(&log_path)?
+ .lines()
+ .map(serde_json::from_str::<Value>)
+ .collect::<Result<Vec<_>, _>>()?;
+ assert_eq!(records.len(), 2);
+ assert_eq!(records[0]["route_id"], "route/passthrough");
+ assert_eq!(records[0]["algorithm"], "passthrough");
+ assert_eq!(records[0]["model"], "model/shared");
+ assert_eq!(records[1]["route_id"], "route/stage");
+ assert_eq!(records[1]["algorithm"], "stage_router");
+ assert_eq!(records[1]["model"], "model/shared");
+ Ok(())
+}
+
#[tokio::test]
async fn routing_log_keeps_the_canonical_session_id_until_a_stream_drains() -> TestResult {
let upstream = MockUpstream::start().await?;
let temp_dir = tempfile::tempdir()?;
+ let log_path = temp_dir.path().join("routing.jsonl");
let state = random_state(&upstream.base_url, &[(ROUTE_MODEL, &["model/a"])])?
- .with_routing_log(temp_dir.path().join("routing.jsonl"))?;
+ .with_routing_log(&log_path)?;
let app = build_switchyard_router(state);
// `send_with_headers` collects the response body, so the stream wrapper reaches
@@ -2589,6 +2666,10 @@ async fn routing_log_keeps_the_canonical_session_id_until_a_stream_drains() -> T
assert_eq!(stats["total_cached_tokens"], 7);
assert_eq!(stats["total_cache_creation_tokens"], 2);
assert_eq!(stats["total_completion_tokens"], 5);
+
+ let record: Value = serde_json::from_str(&std::fs::read_to_string(log_path)?)?;
+ assert_eq!(record["route_id"], ROUTE_MODEL);
+ assert_eq!(record["algorithm"], "random");
Ok(())
}
@@ -3257,6 +3338,8 @@ async fn advisor_route_routing_log_records_classifier_tier() -> TestResult {
.find(|record| record["model"] == "model/advisor")
.ok_or("consult row present")?;
assert_eq!(consult["tier"], "classifier");
+ assert_eq!(consult["route_id"], "switchyard/advisor");
+ assert_eq!(consult["algorithm"], "advisor_gate");
assert_eq!(consult["session_id"], "session-1");
assert_eq!(consult["prompt_tokens"], 40);
Ok(())