Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(server): record route attribution in durable log (#618)
Clement Pakkam Isaac5879c1097b53
diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.mdindex 0641f8308..87c281886 100644--- a/crates/switchyard-server/README.md+++ b/crates/switchyard-server/README.md@@ -93,7 +93,9 @@ for equal weighting. The optional `seed` reproduces the selection sequence for t## Session routing logPass `--routing-log-file PATH` to append one JSON record after each completed routed response.-Streaming responses are recorded after the stream drains. When enabled,+Streaming responses are recorded after the stream drains. Each record names the client-facing+`route_id`, routing `algorithm`, served `model`, tier, and token usage so routes sharing a backend+remain distinguishable. When enabled,`GET /v1/routing/session-stats?session_id=ID` rescans the durable log and returns call and tokentotals for that normalized session ID, normally supplied as `x-switchyard-session-id`, grouped byserved model. The legacy `proxy_x_session_id` remains a fallback when no normalized session ID isdiff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex 7409f3c12..8b6f06de8 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -1005,6 +1005,7 @@ fn resolve_route(Ok((route, request))}+/// Resolves and executes an LLM request, attaching route identity when durable logging is enabled.async fn handle_llm_request(state: ServerState,started: RequestStart,@@ -1018,6 +1019,12 @@ async fn handle_llm_request(Ok(resolved) => resolved,Err(response) => return response,};+ let routing_log_context = routing_log_context.map(|context| {+ context.with_route(+ request.llm_request.model.as_deref().unwrap_or_default(),+ route.algorithm_name(),+ )+ });// Only the Codex namespace mapping is needed downstream, not the whole request.let request_extensions = request.llm_request.extensions.clone();let observer = stats_observer(diff --git a/crates/switchyard-server/src/routing_log.rs b/crates/switchyard-server/src/routing_log.rsindex 47ed65387..2e9dd8637 100644--- a/crates/switchyard-server/src/routing_log.rs+++ b/crates/switchyard-server/src/routing_log.rs@@ -51,6 +51,8 @@ impl RoutingLog {let usage = token_usage(usage);let record = RoutingRecord {ts: format_rfc3339_millis(SystemTime::now()).to_string().into(),+ route_id: context.route_id.into(),+ algorithm: context.algorithm.into(),task: context.task.map(Cow::Owned),trial_id: context.trial_id.map(Cow::Owned),session_id: context.session_id.map(Cow::Owned),@@ -95,9 +97,11 @@ pub(crate) fn snapshot(Ok((snapshot.total_calls > 0).then_some(snapshot))}-/// Request fields retained until terminal usage and routing are available.+/// Holds request metadata until route resolution attaches the durable route identity.#[derive(Clone)]pub(crate) struct RoutingLogContext {+ route_id: String,+ algorithm: String,task: Option<String>,trial_id: Option<String>,session_id: Option<String>,@@ -108,6 +112,8 @@ impl RoutingLogContext {pub(crate) fn from_metadata(metadata: &Metadata) -> Self {let headers = metadata.http_headers.as_ref();Self {+ route_id: String::new(),+ algorithm: String::new(),task: headers.and_then(|headers| nonempty_header(headers, TASK_HEADER)).map(str::to_string),@@ -121,6 +127,13 @@ impl RoutingLogContext {}),}}++ /// Attaches the resolved route identity shared by every log entry for this request.+ pub(crate) fn with_route(mut self, route_id: impl Into<String>, algorithm: &str) -> Self {+ self.route_id = route_id.into();+ self.algorithm = algorithm.to_string();+ self+ }}/// One appended routing record, and the read schema [`snapshot`] parses back,@@ -130,6 +143,8 @@ impl RoutingLogContext {#[serde(default)]struct RoutingRecord<'a> {ts: Cow<'a, str>,+ route_id: Cow<'a, str>,+ algorithm: Cow<'a, str>,#[serde(borrow)]task: Option<Cow<'a, str>>,#[serde(borrow)]diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex 367f94394..43e6abf7f 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -2550,12 +2550,89 @@ async fn routing_log_prefers_canonical_and_preserves_legacy_fallback() -> TestReOk(())}+/// Two routes serving one model must remain distinguishable in the durable accounting record.+#[tokio::test]+async fn routing_log_attributes_shared_models_to_the_requested_route() -> TestResult {+ let upstream = MockUpstream::start().await?;+ let temp_dir = tempfile::tempdir()?;+ let log_path = temp_dir.path().join("routing.jsonl");+ let state = load_test_config(&format!(+ r#"+schema_version = 1++[llm_clients.upstream]+format = "openai_chat"+base_url = "{base_url}"++[targets.shared]+id = "model/shared"+llm_client = "upstream"++[targets.capable]+id = "model/capable"+llm_client = "upstream"++[routes.passthrough]+id = "route/passthrough"+type = "passthrough"+target = "shared"++[routes.stage]+id = "route/stage"+type = "stage_router"+capable_target = "capable"+efficient_target = "shared"+picker = "efficient_first"+confidence_threshold = 1.0+"#,+ base_url = upstream.base_url+ ))?+ .with_routing_log(&log_path)?;+ let app = build_switchyard_router(state);++ for route_id in ["route/passthrough", "route/stage"] {+ let response = send(+ &app,+ "POST",+ "/v1/chat/completions",+ Some(json!({+ "model": route_id,+ "messages": [{"role": "user", "content": "hello"}]+ })),+ )+ .await?;+ assert_eq!(response.status, StatusCode::OK, "{route_id}");+ assert_eq!(+ response+ .headers+ .get("x-model-router-selected-model")+ .and_then(|value| value.to_str().ok()),+ Some("model/shared"),+ "{route_id}"+ );+ }++ let records = std::fs::read_to_string(&log_path)?+ .lines()+ .map(serde_json::from_str::<Value>)+ .collect::<Result<Vec<_>, _>>()?;+ assert_eq!(records.len(), 2);+ assert_eq!(records[0]["route_id"], "route/passthrough");+ assert_eq!(records[0]["algorithm"], "passthrough");+ assert_eq!(records[0]["model"], "model/shared");+ assert_eq!(records[1]["route_id"], "route/stage");+ assert_eq!(records[1]["algorithm"], "stage_router");+ assert_eq!(records[1]["model"], "model/shared");+ Ok(())+}+#[tokio::test]async fn routing_log_keeps_the_canonical_session_id_until_a_stream_drains() -> TestResult {let upstream = MockUpstream::start().await?;let temp_dir = tempfile::tempdir()?;+ let log_path = temp_dir.path().join("routing.jsonl");let state = random_state(&upstream.base_url, &[(ROUTE_MODEL, &["model/a"])])?- .with_routing_log(temp_dir.path().join("routing.jsonl"))?;+ .with_routing_log(&log_path)?;let app = build_switchyard_router(state);// `send_with_headers` collects the response body, so the stream wrapper reaches@@ -2589,6 +2666,10 @@ async fn routing_log_keeps_the_canonical_session_id_until_a_stream_drains() -> Tassert_eq!(stats["total_cached_tokens"], 7);assert_eq!(stats["total_cache_creation_tokens"], 2);assert_eq!(stats["total_completion_tokens"], 5);++ let record: Value = serde_json::from_str(&std::fs::read_to_string(log_path)?)?;+ assert_eq!(record["route_id"], ROUTE_MODEL);+ assert_eq!(record["algorithm"], "random");Ok(())}@@ -3257,6 +3338,8 @@ async fn advisor_route_routing_log_records_classifier_tier() -> TestResult {.find(|record| record["model"] == "model/advisor").ok_or("consult row present")?;assert_eq!(consult["tier"], "classifier");+ assert_eq!(consult["route_id"], "switchyard/advisor");+ assert_eq!(consult["algorithm"], "advisor_gate");assert_eq!(consult["session_id"], "session-1");assert_eq!(consult["prompt_tokens"], 40);Ok(())